From 3c8a69cbe8e158b1a4c12195b28b1c3ce78088e4 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 20:38:56 -0800 Subject: [PATCH 01/63] test(function_node): failing tests for the rows a job node returns after a run (ITL-1093) Co-Authored-By: Claude Opus 5.5 --- ...-11-13-itl-1093-job-node-rows-after-run.md | 87 +++++ .../test_job_node_rows_after_run.py | 300 ++++++++++++++++++ 2 files changed, 387 insertions(+) create mode 100644 superpowers/plans/2026-11-13-itl-1093-job-node-rows-after-run.md create mode 100644 tests/test_pipeline/test_job_node_rows_after_run.py diff --git a/superpowers/plans/2026-11-13-itl-1093-job-node-rows-after-run.md b/superpowers/plans/2026-11-13-itl-1093-job-node-rows-after-run.md new file mode 100644 index 000000000..bc3428932 --- /dev/null +++ b/superpowers/plans/2026-11-13-itl-1093-job-node-rows-after-run.md @@ -0,0 +1,87 @@ +# ITL-1093: a function job node returns the rows of its run's inputs — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** After a run on either orchestrator, a `FunctionJobNode` returns exactly the rows of +that run's inputs (`as_table`, `iter_data`, `head`, `run()`, a consumer built over it), row for +row equal between sync and async. Nothing stored changes. + +**Architecture:** `FunctionJobNode._cached_output_datas` stays a superset cache keyed by base +entry id. A new `_run_entry_ids: list[bytes] | None` records the base entry ids of the last +completed run's inputs in input order (`None` = no completed run in this process). One helper +`_held_pairs()` is the only reader of the held rows: with run ids it yields one row per run +input (not deduplicated, so held rows equal what downstream received), else every held value +(unchanged no-run behaviour). A run that raises defines no rows: the node clears its in-memory +rows and run marker (`clear_cache()`) and reads as a no-run node. + +**Tech Stack:** Python 3.11+, PyArrow, pytest, `uv run`. + +Coordinator decisions (November 13, 2026): proposal approved; no-run read unchanged (option A, +documented; B/C reported to the developer); duplicates not deduplicated; a raised run → no-run +read (option i); REPLAY union and async `fail_fast` (ITL-879) reported only. + +--- + +## Cause (measured on main b4fde330) + +- `async_execute` loads `self._load_cached_entries()` with no ids (every pipeline record under + the node's content hash) into `_cached_output_datas`; every read yields every dict value. +- Sync `execute` loads only this run's ids (`get_cached_results(base_entry_ids)`), so it looked + right only because a new job builds fresh node objects. Re-running the same job object with a + rebound source accumulates on sync too (3/6/6/9). + +## Files + +- Modify: `src/orcapod/core/nodes/function_node.py` (FunctionJobNode: `__init__`, `clear_cache`, + `execute`, `async_execute`, `iter_data`, `head`, new `_held_pairs`) +- Create: `tests/test_pipeline/test_job_node_rows_after_run.py` +- Create: `changelog.d/ITL-1093.md`, `design_issues/ITL-1093-JN1.md` +- Modify: `CLAUDE.md`, `.zed/rules` (Phase 1 sentence); `docs/concepts/` sentence if one exists. + +## Tasks + +### Task 1: Failing tests + +- [ ] Write `tests/test_pipeline/test_job_node_rows_after_run.py`: pipeline `rowwise` (inc over + DictSource source_id "s"), `downstream` (dbl over rowwise), `grouped` (total over + `src.group_by("g")`). Sequence 3 / 6 / 3 again / 3 different rows. + - fresh job per run, sync and async: each node's `as_table`, `iter_data`, `head(100)`, + `run()` and a consumer `dbl.pod(job.nodes["rowwise"])` return the rows of the run's inputs; + sync == async as sets of rows with system tags; the 6-row run reports 3 reused. + - same job object, `job.bind(sources=...)` per run, sync and async: same expectations. + - filtered inputs (function returns None) are in no read. + - a raised run (sync fail_fast ValueError; sync and async StoreWriteError raised part-way) + reads as a no-run node: equal to a fresh compiled job's read over the same store. +- [ ] Run (Python 3.12 and 3.11): `uv run pytest tests/test_pipeline/test_job_node_rows_after_run.py` + — expect async and same-job cases to FAIL. +- [ ] WIP commit. + +### Task 2: Fix + +- [ ] `_run_entry_ids` class default `None`, set in `__init__`; `clear_cache()` resets it. +- [ ] `_held_pairs()` helper; `iter_data` (both branches) and `head`'s held branch read through it; + `head` takes the held branch when `_run_entry_ids is not None` even with no held rows. +- [ ] `execute`: completed flag; at success set `_run_entry_ids = base_entry_ids` and drop the + cached table/hash column; on any raise after processing began, `_commit_after_failure` + (unchanged) then `clear_cache()`. +- [ ] `async_execute` DB path: `route_inputs` appends each input's base entry id to a local + list; at success set `_run_entry_ids`; a raise (Exception or cancellation) → `clear_cache()`. +- [ ] Run new tests (3.12 + 3.11) and targeted node/pipeline tests; commit. + +### Task 3: Docs and records + +- [ ] Docstrings of `iter_data`, `as_table`, `head`, `run`, `execute`, `async_execute` state the rule. +- [ ] `changelog.d/ITL-1093.md` under `### Fixed`; check with + `uv run python scripts/assemble_changelog.py --check`. +- [ ] `design_issues/ITL-1093-JN1.md` (resolved, Fix note); `uv run python scripts/design_issues.py --check`. +- [ ] CLAUDE.md + `.zed/rules` sentence → "After a run, a `FunctionJobNode` returns exactly the rows + of that run's inputs, sync or async (a run that raised defines no rows). Read without a + completed run in this process (fresh or loaded job), it returns every record stored under its + content hash for the current data context. A run skips inputs whose entry is already stored." +- [ ] docs/concepts sentence if one states what a job node returns after a run. +- [ ] ruff, ruff format, basedpyright (exit code), golden suite, targeted suites; commit; push; draft PR. + +## Notes for a successor + +- Probes: scratchpad (not durable); measured tables are in the coordinator ask and the PR body. +- PR #359 (ITL-1063) edits the same `async_execute`/`execute` hunks; semantics compatible. diff --git a/tests/test_pipeline/test_job_node_rows_after_run.py b/tests/test_pipeline/test_job_node_rows_after_run.py new file mode 100644 index 000000000..7f2ac88c8 --- /dev/null +++ b/tests/test_pipeline/test_job_node_rows_after_run.py @@ -0,0 +1,300 @@ +"""After a run, a function job node returns exactly the rows of that run's inputs (ITL-1093). + +One store, a source of one ``source_id`` holding 3 rows, then 6, then the same 3 again, then +3 different rows. After each run, on either orchestrator, every function job node returns +the rows of that run's inputs through ``as_table``, ``iter_data``, ``head`` and ``run()``, +and so does a consumer built over the node afterwards; the two orchestrators agree row for +row. Stored results of other inputs stay stored and reusable. A run that raised defines no +rows: the node then reads as a node with no run in this process (every row stored under +its content hash). +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.errors import StoreWriteError +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.sources import DictSource + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + """Return ``x + 1``.""" + return x + 1 + + +@function_pod(output_keys="z") +def dbl(y: int) -> int: + """Return ``2 * y``.""" + return 2 * y + + +@function_pod(output_keys="total") +def total(x: list[int]) -> int: + """Return the sum of a group's values.""" + return sum(x) + + +@function_pod(output_keys="y") +def null_on_10(x: int) -> int | None: + """Return ``x + 1``, or a null value for ``x == 10``.""" + return None if x == 10 else x + 1 + + +@function_pod(output_keys="y") +def fails_on_20(x: int) -> int: + """Return ``x + 1``; raise for ``x == 20``.""" + if x == 20: + raise ValueError("x is 20") + return x + 1 + + +@function_pod(output_keys="y") +def store_fails_on_20(x: int) -> int: + """Return ``x + 1``; for ``x == 20`` raise the error a failed store write raises. + + A ``StoreError`` fails the run whatever the error policy, on both orchestrators, as a + failed write of the node's results does. + """ + if x == 20: + raise StoreWriteError("simulated failed store write") + return x + 1 + + +ROWS = [{"g": i % 2, "k": i, "x": 10 * i} for i in range(6)] +# 3 rows, 6 rows (3 more), the same 3 again (fewer), 3 different rows. The source's record +# ids are positional, so the last run's rows are new inputs whose results are stored. +SEQUENCE = [ROWS[:3], ROWS, ROWS[:3], ROWS[3:]] +ORCHESTRATORS = ["sync", "async"] + + +def _source(data: list[dict[str, int]]) -> DictSource: + return DictSource(data, tag_columns=["g", "k"], source_id="s") + + +def _build(data: list[dict[str, int]], db: Any) -> PipelineJob: + job = PipelineJob(name="itl1093", database=db) + with job: + rowwise = inc.pod(_source(data), label="rowwise") + dbl.pod(rowwise, label="downstream") + total.pod(_source(data).group_by("g"), label="grouped") + return job + + +def _orchestrator(kind: str, error_policy: str = "continue") -> Any: + if kind == "sync": + return SyncPipelineOrchestrator(error_policy=error_policy) # pyright: ignore[reportArgumentType] + return AsyncPipelineOrchestrator(error_policy=error_policy) # pyright: ignore[reportArgumentType] + + +def _canonical(rows: list[dict[str, Any]]) -> list[tuple[tuple[str, str], ...]]: + """Rows as a sorted list, each a sorted tuple of (column, repr(value)).""" + return sorted(tuple(sorted((k, repr(v)) for k, v in row.items())) for row in rows) + + +ALL = {"system_tags": True} + + +def _reads(node: Any) -> dict[str, list[tuple[tuple[str, str], ...]]]: + """Every public read of ``node``, rows with system tags, canonical.""" + iterated = [ + {**tag.as_dict(all_info=True), **data.as_dict()} for tag, data in node.iter_data() + ] + return { + "as_table": _canonical(node.as_table(columns=ALL).to_pylist()), + "head": _canonical(node.head(100, columns=ALL).as_table().to_pylist()), + "iter_data": _canonical( + [ + {k: v for k, v in row.items() if not k.startswith("__")} + for row in iterated + ] + ), + } + + +def _values(node: Any, key: Callable[[dict[str, Any]], Any]) -> list[Any]: + return sorted(key(row) for row in node.as_table().to_pylist()) + + +def _expected_rowwise(data: list[dict[str, int]]) -> list[tuple[int, int]]: + return sorted((row["k"], row["x"] + 1) for row in data) + + +def _expected_downstream(data: list[dict[str, int]]) -> list[tuple[int, int]]: + return sorted((row["k"], 2 * (row["x"] + 1)) for row in data) + + +def _expected_grouped(data: list[dict[str, int]]) -> list[tuple[int, int]]: + sums: dict[int, int] = {} + for row in data: + sums[row["g"]] = sums.get(row["g"], 0) + row["x"] + return sorted(sums.items()) + + +def _check_run(job: PipelineJob, data: list[dict[str, int]]) -> None: + """Assert every node of ``job`` returns the rows of ``data``, through every read.""" + nodes = job.nodes + assert _values(nodes["rowwise"], lambda r: (r["k"], r["y"])) == _expected_rowwise( + data + ) + assert _values(nodes["downstream"], lambda r: (r["k"], r["z"])) == ( + _expected_downstream(data) + ) + assert _values(nodes["grouped"], lambda r: (r["g"], r["total"])) == ( + _expected_grouped(data) + ) + for label in ("rowwise", "downstream", "grouped"): + reads = _reads(nodes[label]) + assert reads["head"] == reads["as_table"], label + assert len(reads["iter_data"]) == len(reads["as_table"]), label + # A consumer built over the node after the run receives the run's rows. + consumer = dbl.pod(nodes["rowwise"]) + assert sorted((r["k"], r["z"]) for r in consumer.as_table().to_pylist()) == ( + _expected_downstream(data) + ) + + +def _snapshot(job: PipelineJob) -> dict[str, Any]: + return {label: _reads(job.nodes[label]) for label in ("rowwise", "downstream", "grouped")} + + +class TestFreshJobPerRun: + """A new job over the same store for every run, as the issue's reproduction.""" + + @pytest.mark.parametrize("kind", ORCHESTRATORS) + def test_each_run_returns_its_inputs_rows(self, kind: str) -> None: + db = InMemoryArrowDatabase() + for index, data in enumerate(SEQUENCE): + job = _build(data, db) + summary = job.run(orchestrator=_orchestrator(kind)) + _check_run(job, data) + if index == 1: + # The 6-row run reuses the 3 stored results of the first run. + assert summary.nodes["rowwise"].reused == 3 + assert summary.nodes["rowwise"].computed == 3 + + def test_sync_and_async_agree_row_for_row(self) -> None: + snapshots: dict[str, list[dict[str, Any]]] = {} + for kind in ORCHESTRATORS: + db = InMemoryArrowDatabase() + snapshots[kind] = [] + for data in SEQUENCE: + job = _build(data, db) + job.run(orchestrator=_orchestrator(kind)) + snapshots[kind].append(_snapshot(job)) + assert snapshots["sync"] == snapshots["async"] + + @pytest.mark.parametrize("kind", ORCHESTRATORS) + def test_node_run_returns_its_inputs_rows(self, kind: str) -> None: + db = InMemoryArrowDatabase() + for data in SEQUENCE: + job = _build(data, db) + job.run(orchestrator=_orchestrator(kind)) + view = job.nodes["rowwise"].run() + assert sorted((r["k"], r["y"]) for r in view.as_table().to_pylist()) == ( + _expected_rowwise(data) + ) + + +class TestSameJobRebound: + """One job object, its source rebound before every run: the same node objects run again.""" + + @pytest.mark.parametrize("kind", ORCHESTRATORS) + def test_each_run_returns_its_inputs_rows(self, kind: str) -> None: + db = InMemoryArrowDatabase() + job = _build(SEQUENCE[0], db) + for data in SEQUENCE: + job.bind(sources={"s": _source(data)}) + job.run(orchestrator=_orchestrator(kind)) + _check_run(job, data) + + def test_sync_and_async_agree_row_for_row(self) -> None: + snapshots: dict[str, list[dict[str, Any]]] = {} + for kind in ORCHESTRATORS: + db = InMemoryArrowDatabase() + job = _build(SEQUENCE[0], db) + snapshots[kind] = [] + for data in SEQUENCE: + job.bind(sources={"s": _source(data)}) + job.run(orchestrator=_orchestrator(kind)) + snapshots[kind].append(_snapshot(job)) + assert snapshots["sync"] == snapshots["async"] + + +def _single(fn: Any, data: list[dict[str, int]], db: Any) -> PipelineJob: + job = PipelineJob(name="itl1093_single", database=db) + with job: + fn.pod(_source(data), label="node") + return job + + +class TestNullAndInactive: + """A null value is a row; an inactive function's inputs (no output) are rows of no read.""" + + @pytest.mark.parametrize("kind", ORCHESTRATORS) + def test_null_value_is_a_row(self, kind: str) -> None: + db = InMemoryArrowDatabase() + _single(null_on_10, ROWS, db).run(orchestrator=_orchestrator(kind)) + job = _single(null_on_10, ROWS[:3], db) + job.run(orchestrator=_orchestrator(kind)) + node = job.nodes["node"] + assert _values(node, lambda r: (r["k"], r["y"])) == [(0, 1), (1, None), (2, 21)] + reads = _reads(node) + assert reads["head"] == reads["as_table"] + assert len(reads["iter_data"]) == 3 + + @pytest.mark.parametrize("kind", ORCHESTRATORS) + def test_inactive_function_gives_no_rows(self, kind: str) -> None: + @function_pod(output_keys="y") + def inactive_inc(x: int) -> int: + return x + 1 + + db = InMemoryArrowDatabase() + _single(inactive_inc, ROWS[3:], db).run(orchestrator=_orchestrator(kind)) + # A function switched off produces no output for the inputs it is given. + inactive_inc.pod.data_function.set_active(False) + job = _single(inactive_inc, ROWS[:3], db) + job.run(orchestrator=_orchestrator(kind)) + node = job.nodes["node"] + assert node.as_table().num_rows == 0 + assert list(node.iter_data()) == [] + assert node.head(100).as_table().num_rows == 0 + + +class TestRaisedRun: + """A run that raised defines no rows: the node reads as a node with no run.""" + + @staticmethod + def _no_run_read(fn: Any, data: list[dict[str, int]], db: Any) -> list[Any]: + fresh = _single(fn, data, db) + fresh.compile() + return _values(fresh.nodes["node"], lambda r: (r["k"], r["y"])) + + @pytest.mark.parametrize( + ("kind", "fn", "policy"), + [ + ("sync", fails_on_20, "fail_fast"), + ("sync", store_fails_on_20, "continue"), + ("async", store_fails_on_20, "continue"), + ], + ) + def test_raised_run_reads_as_no_run(self, kind: str, fn: Any, policy: str) -> None: + db = InMemoryArrowDatabase() + # Store the results of rows 3..5 first, then run rows 0..3, which fails at x=20. + _single(fn, ROWS[3:], db).run(orchestrator=_orchestrator(kind)) + job = _single(fn, ROWS[:4], db) + with pytest.raises((ValueError, StoreWriteError)): + job.run(orchestrator=_orchestrator(kind, policy)) + node = job.nodes["node"] + expected = self._no_run_read(fn, ROWS[:4], db) + assert _values(node, lambda r: (r["k"], r["y"])) == expected + reads = _reads(node) + assert reads["head"] == reads["as_table"] + # Nothing stored was dropped: the stored rows of the earlier run are among them. + assert {(3, 31), (4, 41), (5, 51)} <= set(expected) From 2e9972e0ca0e2250d8c17c36f847d78869088968 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 20:51:24 -0800 Subject: [PATCH 02/63] fix(function_node): after a run a job node returns exactly the rows of that run's inputs (ITL-1093) The node records the base entry ids of its last completed run's inputs and every read of its rows goes through one helper that keeps only those inputs' rows. The async run loads every stored record into the cache, so it returned every row ever stored for the node; a re-run of the same node objects did so on the sync path too. A run that raised defines no rows. Nothing stored, no hash, no layout changes. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 7 +- CLAUDE.md | 8 +- changelog.d/ITL-1093.md | 3 + design_issues/ITL-1093-JR1.md | 26 ++++ docs/concepts/function-pods.md | 12 +- src/orcapod/core/nodes/function_node.py | 136 +++++++++++++----- .../test_job_node_rows_after_run.py | 99 +++++++++++-- 7 files changed, 236 insertions(+), 55 deletions(-) create mode 100644 changelog.d/ITL-1093.md create mode 100644 design_issues/ITL-1093-JR1.md diff --git a/.zed/rules b/.zed/rules index 0342f7342..2d923a544 100644 --- a/.zed/rules +++ b/.zed/rules @@ -773,8 +773,11 @@ data. The rules: by the user is never changed. SourceStreamBuilder.build has no bare-hash default: it needs a source_id or a source_kind. - ArrowTableStream requires at least one data column; raises ValueError otherwise. -- FunctionNode Phase 1 returns ALL records in the shared pipeline_path DB table. - Phase 2 skips inputs whose hash is already in the DB. +- After a run, a FunctionJobNode returns exactly the rows of that run's inputs, sync or + async (ITL-1093; a run that raised defines no rows). Read without a completed run in this + process (a fresh or loaded job), it returns every record stored under its content hash for + the current data context, not only its current inputs. A run skips inputs whose entry is + already stored. - Empty data → ArrowTableSource raises ValueError("Table is empty"). - DerivedSource over a job node before run() → presents zero rows with the origin's columns (no computed records; no error), in the plain form and with tags_as_data. diff --git a/CLAUDE.md b/CLAUDE.md index 8164ba5f9..d075910b4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -814,9 +814,11 @@ data. The rules: by the user is never changed. `SourceStreamBuilder.build` has no bare-hash default: it needs a `source_id` or a `source_kind`. - `ArrowTableStream` requires at least one data column; raises `ValueError` otherwise. -- `FunctionNode.iter_data()` Phase 1 returns ALL records in the shared `pipeline_path` - DB table (not filtered to current inputs). Phase 2 skips inputs whose hash is already - in the DB. +- After a run, a `FunctionJobNode` returns exactly the rows of that run's inputs, sync or + async (ITL-1093; a run that raised defines no rows). Read without a completed run in this + process (a fresh or loaded job), it returns every record stored under its content hash for + the current data context, not only its current inputs. A run skips inputs whose entry is + already stored. - Empty data → `ArrowTableSource` raises `ValueError("Table is empty")`. - `DerivedSource` over a job node before `run()` → presents zero rows with the origin's columns (no computed records; no error), in the plain form and with `tags_as_data`. diff --git a/changelog.d/ITL-1093.md b/changelog.d/ITL-1093.md new file mode 100644 index 000000000..2e4421eb8 --- /dev/null +++ b/changelog.d/ITL-1093.md @@ -0,0 +1,3 @@ +### Fixed + +- After `job.run()` on either orchestrator, a function job node's `as_table()`, `iter_data()`, `head()` and `run()`, and a stream built over the node, return exactly the rows of that run's inputs (ITL-1093). Before, after an async run a node returned every row stored under its key: a pipeline whose source rows changed between runs under one `source_id` showed the results of earlier runs too (for a function over `group_by`, the partial groups of an earlier run next to the full ones; rows of source rows no longer present). The sync orchestrator did the same when one job object ran again with its source rebound. Rows sent downstream during the run and the `RunSummary` were already right. Results of other inputs stay stored and are reused when those inputs come back; nothing stored, no hash and no table layout changes. A run that raised defines no rows: the node then reads as a node with no run in this process. A node read with no finished run in this process (a job built but not run, or a loaded job) still returns every row stored under its key. diff --git a/design_issues/ITL-1093-JR1.md b/design_issues/ITL-1093-JR1.md new file mode 100644 index 000000000..e270f4ffc --- /dev/null +++ b/design_issues/ITL-1093-JR1.md @@ -0,0 +1,26 @@ +## `src/orcapod/core/nodes/function_node.py` + +### JR1 — After a run, a function job node returned rows of other runs' inputs +**Status:** resolved +**Severity:** high +**Issue:** ITL-1093 + +A `FunctionJobNode` kept one in-memory cache of results keyed by base entry id, and every read +(`iter_data`, and through it `as_table`, `run()` and any consumer; `head`) returned all of it. +Nothing recorded which inputs the run processed. `async_execute` loads every pipeline record +stored under the node's content hash into that cache before routing its inputs, so after an +async run the node returned every row ever stored for it: partial groups of an earlier run next +to full ones, rows of source rows no longer present. The sync `execute` loaded only its inputs' +entries, so it looked right with a new job per run, but a cache is add-only: running the same +node objects again (one job, its source rebound) accumulated rows on the sync path too. Rows sent +downstream during the run and the `RunSummary` were right on both paths. + +**Fix:** The node records the base entry ids of its last completed run's inputs +(`_run_entry_counts`, set by `execute` and `async_execute`), and every read of its rows goes +through one helper, `_held_pairs`, which keeps only those inputs' rows (one per input, so two +equal inputs give two rows, as downstream received them). The cache stays a superset: stored +results of other inputs stay stored and reusable. A run that raised defines no rows: the node +clears its in-memory rows and reads as a node with no run in this process (every row stored +under its content hash), on both orchestrators. Nothing stored, no hash and no layout changed. +A read with no completed run in this process (a fresh or loaded job) still returns every row +stored under the node's content hash. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 108453b24..5f414b039 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -333,6 +333,13 @@ database. Each recorded step becomes a node, available afterwards as `job.nodes[ listed under its `source_id`). A node is a stream, so `as_table()`, `iter_data()` and `output_schema()` all work on it. +After `job.run()`, a function node returns exactly the rows of that run's inputs, on either +orchestrator. The database may hold more under the same node: results of an earlier run +over other rows of the source stay stored and are reused when those rows come back, but +they are not rows of this run. A run that raised defines no rows. A node read with no +finished run in this process (a job built but not run, or a loaded job) returns every row +stored under its key. + `InMemoryArrowDatabase` lasts only as long as the Python process. Use `DeltaTableDatabase("some/directory")` from `orcapod.databases` to keep results on disk between sessions. @@ -696,8 +703,9 @@ file names each run's rows by the content hash the run stored them under, so a l reads exactly what the same node built from your code would read from that store, and no rows under that key is an empty result, not an error. -- A function node's `as_table()`, `iter_data()` and `get_all_records()` return the rows the - saved run stored, with the types they were stored with. Rows come in the order the store +- A function node's `as_table()`, `iter_data()` and `get_all_records()` return the rows + stored under the saved run's key, with the types they were stored with: those of every + run of that node over the same sources, not only the saved run's inputs. Rows come in the order the store returns them; sort them if order matters. - Operators (a join, a filter) store outputs only when caching is enabled, under `odb_v1`. The saved file does not record an operator's cache mode (follow-up ITL-977), so attach a diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 186a8e384..aa9c50dff 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -21,7 +21,8 @@ import logging import uuid from abc import abstractmethod -from collections.abc import Callable, Iterator, Mapping, Sequence +from collections import Counter +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Literal, NamedTuple, cast from orcapod import contexts @@ -1054,6 +1055,10 @@ class FunctionJobNode(FunctionNodeBase): _current_tables_checked: bool = False # Set once a store that cannot list its tables was reported. _unlisted_store_reported: bool = False + # Base entry ids of the inputs of the last completed run in this process, each with + # its number of inputs; None when no run completed (ITL-1093). A class default so + # nodes rebuilt from a saved descriptor start without a run. + _run_entry_counts: Counter[bytes] | None = None def __init__( self, @@ -1083,6 +1088,7 @@ def __init__( ] = {} self._cached_output_table: "pa.Table | None" = None self._cached_content_hash_column: "pa.Array | None" = None + self._run_entry_counts = None # DB persistence state (initially None; set via __init__ params or attach_databases) self._pipeline_database: ArrowDatabaseProtocol | None = None @@ -1505,11 +1511,44 @@ def from_descriptor( # ------------------------------------------------------------------ def clear_cache(self) -> None: - """Clear in-memory output caches, content hash cache, and node identity path cache.""" + """Clear in-memory output caches, content hash cache, and node identity path cache. + + The node then holds no run: it reads as a node with no completed run in this + process (every row stored under its content hash). + """ super().clear_cache() self._cached_output_datas.clear() self._cached_output_table = None self._cached_content_hash_column = None + self._run_entry_counts = None + + def _held_pairs(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: + """Yield the (tag, data) rows this node holds; every read of them goes through here. + + After a completed run, the rows are those of that run's inputs: one per input + (two equal inputs give two rows, as downstream received them). The in-memory + cache may hold more (results of other inputs, read from the store), which stay + cached for reuse but are not rows of this run (ITL-1093). Without a completed + run in this process, every held row. Rows come in the order the cache holds + them. An input without output (an inactive function) is not a row. + """ + held = self._cached_output_datas + counts = self._run_entry_counts + if counts is None: + pairs: Iterable[tuple[TagProtocol, DataProtocol | None]] = held.values() + else: + pairs = ( + pair for eid, pair in held.items() for _ in range(counts.get(eid, 0)) + ) + for tag, data in pairs: + if data is not None: + yield tag, data + + def _set_run_rows(self, entry_ids: list[bytes]) -> None: + """Make the inputs of a completed run, by base entry id, the rows this node holds.""" + self._run_entry_counts = Counter(entry_ids) + self._cached_output_table = None + self._cached_content_hash_column = None def set_ephemeral_store(self, store: "ArrowDatabaseProtocol | None") -> None: """Assign or remove the ephemeral result store. @@ -1872,6 +1911,11 @@ def execute( (populated from DB if needed), compute if missing, fire ``on_data_end`` or ``on_data_crash``. + Afterwards the node's rows (``iter_data``, ``as_table``, ``head``) are those + of this run's inputs. If the run raises, it defines no rows: the node drops + its in-memory rows and reads as a node with no run in this process; what was + stored stays stored. + Args: input_stream: The input stream to process. observer: Optional execution observer for hooks. @@ -1910,23 +1954,24 @@ def execute( tag_schema = input_stream.output_schema(columns={"system_tags": True})[0] ctx_obs.on_node_start(node_label, node_hash, tag_schema=tag_schema) - # Collect upstream entries and resolve base_entry_ids (stable across recomputation) - upstream_entries: list[tuple[TagProtocol, DataProtocol, bytes]] = [ - (tag, data, self.compute_base_entry_id(tag, data)) - for tag, data in input_stream.iter_data() - ] - base_entry_ids = [eid for _, _, eid in upstream_entries] - - # Hot-load any already-computed results from DB into _cached_output_datas. - # get_cached_results() is called for its side effect (populating the - # in-memory cache); the returned dict is intentionally discarded here so - # that the per-data cache-hit check below uses _cached_output_datas - # directly — which includes None-output entries (function returned None) - # and prevents spurious recomputation of already-processed data. - self.get_cached_results(base_entry_ids=base_entry_ids) - output: list[tuple[TagProtocol, DataProtocol]] = [] try: + # Collect upstream entries and resolve base_entry_ids (stable across + # recomputation) + upstream_entries: list[tuple[TagProtocol, DataProtocol, bytes]] = [ + (tag, data, self.compute_base_entry_id(tag, data)) + for tag, data in input_stream.iter_data() + ] + base_entry_ids = [eid for _, _, eid in upstream_entries] + + # Hot-load any already-computed results from DB into _cached_output_datas. + # get_cached_results() is called for its side effect (populating the + # in-memory cache); the returned dict is intentionally discarded here so + # that the per-data cache-hit check below uses _cached_output_datas + # directly — which includes None-output entries (function returned None) + # and prevents spurious recomputation of already-processed data. + self.get_cached_results(base_entry_ids=base_entry_ids) + for tag, data, base_entry_id in upstream_entries: ctx_obs.on_data_start(node_label, tag, data) @@ -1993,8 +2038,15 @@ def execute( ) except Exception as run_error: self._commit_after_failure(run_error) + # A run that raised defines no rows: the node reads as one with no run. + self.clear_cache() + raise + except BaseException: + self.clear_cache() raise + # The node's rows are those of this run's inputs, whatever else it caches. + self._set_run_rows(base_entry_ids) ctx_obs.on_node_end(node_label, node_hash) # Mark this node as freshly computed so subsequent iter_data() calls # skip the is_stale check and serve results directly from the in-memory cache. @@ -3086,8 +3138,8 @@ def head( ) -> StreamView: """Return the first rows the node holds or has stored; never runs the function. - After a run (or ``execute``) the rows are the node's results in memory, the - first of what ``as_table`` returns. Otherwise they are the first of its stored + After a run (or ``execute``) the rows are the node's results in memory for that + run's inputs, the first of what ``as_table`` returns. Otherwise they are the first of its stored rows: the node's whole stored slice is read and nothing is kept on the node. With nothing stored, there are no rows. The node has all of its rows, so with ``sort_by_tags`` (or ``all_info``) the head is exactly @@ -3107,9 +3159,11 @@ def head( config, ViewStatus.UNAVAILABLE, note="no store attached" ) current = status == LoadStatus.CACHE_ONLY or not self.is_stale - if current and self._cached_output_datas: + if current and ( + self._run_entry_counts is not None or self._cached_output_datas + ): return self._head_of_pairs( - list(self._cached_output_datas.values()), n, config, origin="held" + list(self._held_pairs()), n, config, origin="held" ) return self._head_from_store(n, config) @@ -3197,13 +3251,17 @@ def as_table( return super().as_table(columns=columns, all_info=all_info) def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: - """Yield all computed (tag, data) pairs for this node. + """Yield this node's (tag, data) rows. Strictly read-only — never triggers computation. Callers must call ``run()`` or ``execute()`` first if they want results computed. - On the first call with an empty in-memory store and a DB attached, - hot-loads all existing records from the DB (one-shot, no recompute). + After a completed run (``execute``, ``async_execute``, ``run()`` or a job run) + the rows are exactly those of that run's inputs, on either orchestrator, even + though results of other inputs stay cached and stored (ITL-1093). A run that + raised defines no rows. Without a completed run in this process, on the first + call with an empty in-memory store and a DB attached, hot-loads every record + stored under the node's content hash (one-shot, no recompute) and yields them. Raises: RuntimeError: If ``load_status`` is UNAVAILABLE. @@ -3222,11 +3280,7 @@ def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: if loaded: self._cached_output_table = None self._cached_content_hash_column = None - yield from ( - (tag, pkt) - for tag, pkt in self._cached_output_datas.values() - if pkt is not None - ) + yield from self._held_pairs() return # FULL / READ_ONLY — in-memory store may be populated from computation @@ -3245,11 +3299,7 @@ def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: self._cached_output_table = None self._cached_content_hash_column = None - yield from ( - (tag, pkt) - for tag, pkt in self._cached_output_datas.values() - if pkt is not None - ) + yield from self._held_pairs() def run(self) -> StreamView: """Eagerly compute all input data, filling pipeline and result databases, and @@ -3312,6 +3362,11 @@ async def async_execute( are served directly from persistent storage without touching the input channel. + Afterwards the node's rows are those of this run's inputs, as after + ``execute``, although every stored result was loaded into its cache. If the + run raises or is cancelled, it defines no rows: the node drops its in-memory + rows and reads as a node with no run in this process. + Args: input_channel: Single readable channel of (tag, data) pairs. output: Writable channel for output (tag, data) pairs. @@ -3405,12 +3460,15 @@ async def async_execute( # Local dict: correlation_key → (original_tag, original_input_data) input_store: dict[bytes, tuple[TagProtocol, DataProtocol, bytes]] = {} + # Base entry ids of this run's inputs, in arrival order (ITL-1093). + run_entry_ids: list[bytes] = [] async def route_inputs() -> None: """Stage 1: send cache hits to output; stamp misses for computation.""" try: async for tag, data in input_channel: base_entry_id = self.compute_base_entry_id(tag, data) + run_entry_ids.append(base_entry_id) # A cached EmptyData is a result that is gone (an # ephemeral result from an earlier run): compute the row # again, as the sync path does (ITL-923). @@ -3589,6 +3647,9 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: tg.create_task(record_and_forward()) # Commit the last rows inside a job (ITL-919); a no-op otherwise. self._commit_at_node_end() + # The node's rows are those of this run's inputs, as the sync path's: + # the cache also holds every stored result loaded above. + self._set_run_rows(run_entry_ids) else: # No-DB path: delegate dispatch entirely to the pod. The pod closes # the writer it is given; that close waits for the finally below, so @@ -3607,6 +3668,13 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: self._update_modified_time() except Exception as run_error: self._commit_after_failure(run_error) + # A run that raised defines no rows: the node reads as one with no run, + # as on the sync path. + self.clear_cache() + raise + except BaseException: + # Cancelled part-way: no rows either. + self.clear_cache() raise finally: await output.close() diff --git a/tests/test_pipeline/test_job_node_rows_after_run.py b/tests/test_pipeline/test_job_node_rows_after_run.py index 7f2ac88c8..fd57128b0 100644 --- a/tests/test_pipeline/test_job_node_rows_after_run.py +++ b/tests/test_pipeline/test_job_node_rows_after_run.py @@ -17,6 +17,7 @@ import pytest from orcapod import PipelineJob, function_pod +from orcapod.core.data_function import PythonDataFunction from orcapod.databases import InMemoryArrowDatabase from orcapod.errors import StoreWriteError from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator @@ -63,7 +64,7 @@ def store_fails_on_20(x: int) -> int: failed write of the node's results does. """ if x == 20: - raise StoreWriteError("simulated failed store write") + raise StoreWriteError(("simulated", "results")) return x + 1 @@ -104,7 +105,8 @@ def _canonical(rows: list[dict[str, Any]]) -> list[tuple[tuple[str, str], ...]]: def _reads(node: Any) -> dict[str, list[tuple[tuple[str, str], ...]]]: """Every public read of ``node``, rows with system tags, canonical.""" iterated = [ - {**tag.as_dict(all_info=True), **data.as_dict()} for tag, data in node.iter_data() + {**tag.as_dict(all_info=True), **data.as_dict()} + for tag, data in node.iter_data() ] return { "as_table": _canonical(node.as_table(columns=ALL).to_pylist()), @@ -161,7 +163,10 @@ def _check_run(job: PipelineJob, data: list[dict[str, int]]) -> None: def _snapshot(job: PipelineJob) -> dict[str, Any]: - return {label: _reads(job.nodes[label]) for label in ("rowwise", "downstream", "grouped")} + return { + label: _reads(job.nodes[label]) + for label in ("rowwise", "downstream", "grouped") + } class TestFreshJobPerRun: @@ -227,10 +232,10 @@ def test_sync_and_async_agree_row_for_row(self) -> None: assert snapshots["sync"] == snapshots["async"] -def _single(fn: Any, data: list[dict[str, int]], db: Any) -> PipelineJob: +def _single(pod: Any, data: list[dict[str, int]], db: Any) -> PipelineJob: job = PipelineJob(name="itl1093_single", database=db) with job: - fn.pod(_source(data), label="node") + pod(_source(data), label="node") return job @@ -240,8 +245,8 @@ class TestNullAndInactive: @pytest.mark.parametrize("kind", ORCHESTRATORS) def test_null_value_is_a_row(self, kind: str) -> None: db = InMemoryArrowDatabase() - _single(null_on_10, ROWS, db).run(orchestrator=_orchestrator(kind)) - job = _single(null_on_10, ROWS[:3], db) + _single(null_on_10.pod, ROWS, db).run(orchestrator=_orchestrator(kind)) + job = _single(null_on_10.pod, ROWS[:3], db) job.run(orchestrator=_orchestrator(kind)) node = job.nodes["node"] assert _values(node, lambda r: (r["k"], r["y"])) == [(0, 1), (1, None), (2, 21)] @@ -252,14 +257,16 @@ def test_null_value_is_a_row(self, kind: str) -> None: @pytest.mark.parametrize("kind", ORCHESTRATORS) def test_inactive_function_gives_no_rows(self, kind: str) -> None: @function_pod(output_keys="y") - def inactive_inc(x: int) -> int: + def add_one(x: int) -> int: return x + 1 db = InMemoryArrowDatabase() - _single(inactive_inc, ROWS[3:], db).run(orchestrator=_orchestrator(kind)) + _single(add_one.pod, ROWS[3:], db).run(orchestrator=_orchestrator(kind)) # A function switched off produces no output for the inputs it is given. - inactive_inc.pod.data_function.set_active(False) - job = _single(inactive_inc, ROWS[:3], db) + data_function: object = add_one.pod.data_function + assert isinstance(data_function, PythonDataFunction) + data_function.set_active(False) + job = _single(add_one.pod, ROWS[:3], db) job.run(orchestrator=_orchestrator(kind)) node = job.nodes["node"] assert node.as_table().num_rows == 0 @@ -272,7 +279,7 @@ class TestRaisedRun: @staticmethod def _no_run_read(fn: Any, data: list[dict[str, int]], db: Any) -> list[Any]: - fresh = _single(fn, data, db) + fresh = _single(fn.pod, data, db) fresh.compile() return _values(fresh.nodes["node"], lambda r: (r["k"], r["y"])) @@ -287,8 +294,8 @@ def _no_run_read(fn: Any, data: list[dict[str, int]], db: Any) -> list[Any]: def test_raised_run_reads_as_no_run(self, kind: str, fn: Any, policy: str) -> None: db = InMemoryArrowDatabase() # Store the results of rows 3..5 first, then run rows 0..3, which fails at x=20. - _single(fn, ROWS[3:], db).run(orchestrator=_orchestrator(kind)) - job = _single(fn, ROWS[:4], db) + _single(fn.pod, ROWS[3:], db).run(orchestrator=_orchestrator(kind)) + job = _single(fn.pod, ROWS[:4], db) with pytest.raises((ValueError, StoreWriteError)): job.run(orchestrator=_orchestrator(kind, policy)) node = job.nodes["node"] @@ -298,3 +305,67 @@ def test_raised_run_reads_as_no_run(self, kind: str, fn: Any, policy: str) -> No assert reads["head"] == reads["as_table"] # Nothing stored was dropped: the stored rows of the earlier run are among them. assert {(3, 31), (4, 41), (5, 51)} <= set(expected) + + +class TestDuplicateInputs: + """Two inputs with one base entry id give two held rows, as downstream receives them. + + Only a stream without system tags can carry two such inputs (a source gives every + row its own record id), so the node is built directly over an ``ArrowTableStream``. + """ + + @staticmethod + def _node() -> Any: + import pyarrow as pa + + from orcapod.core.nodes import FunctionJobNode + from orcapod.core.streams import ArrowTableStream + + schema = pa.schema( + [ + pa.field("k", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] + ) + table = pa.Table.from_arrays( + [pa.array([0, 0, 2], pa.int64()), pa.array([5, 5, 7], pa.int64())], + schema=schema, + ) + return FunctionJobNode( + inc.pod, # pyright: ignore[reportArgumentType] # the decorator's pod type is not declared as FunctionPodProtocol + ArrowTableStream(table, tag_columns=["k"]), + pipeline_database=InMemoryArrowDatabase(), + result_database=InMemoryArrowDatabase(), + ) + + def test_sync(self) -> None: + node = self._node() + view = node.run() + held = sorted((r["k"], r["y"]) for r in node.as_table().to_pylist()) + assert held == [(0, 6), (0, 6), (2, 8)] + assert sorted((r["k"], r["y"]) for r in view.as_table().to_pylist()) == held + + def test_async(self) -> None: + import asyncio + + from orcapod.channels import Channel + + node = self._node() + + async def run() -> list[tuple[int, int]]: + inputs = Channel(buffer_size=16) + outputs = Channel(buffer_size=16) + # The input stream is the node's own upstream (a test of async_execute). + for item in node.upstreams[0].iter_data(): + await inputs.writer.send(item) + await inputs.writer.close() + await node.async_execute(inputs.reader, outputs.writer) + got = [ + (tag.as_dict()["k"], data.as_dict()["y"]) + async for tag, data in outputs.reader + ] + return sorted(got) + + received = asyncio.run(run()) + assert received == [(0, 6), (0, 6), (2, 8)] + assert sorted((r["k"], r["y"]) for r in node.as_table().to_pylist()) == received From 51cb1b5c90963a876ea2376fe3dcde9d6e7f22fd Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:20:37 -0800 Subject: [PATCH 03/63] docs(plans): ITL-1095 diagnosis and proposal (WIP) Co-Authored-By: Claude Opus 5.5 --- ...26-11-13-itl-1095-node-current-run-view.md | 172 +++++++++++++++ .../plans/2026-11-13-itl-1095-proposal.md | 196 ++++++++++++++++++ 2 files changed, 368 insertions(+) create mode 100644 superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md create mode 100644 superpowers/plans/2026-11-13-itl-1095-proposal.md diff --git a/superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md b/superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md new file mode 100644 index 000000000..de97f4096 --- /dev/null +++ b/superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md @@ -0,0 +1,172 @@ +# ITL-1095: a job node shows the current run; load-only run + +Status: DIAGNOSIS + PROPOSAL sent to the coordinator (ask), waiting for decisions. No code changed. +Base: PR #378 branch head `2e9972e0` (ITL-1093). Branch: `eywalker/itl-1095-design-what-a-job-node-shows-the-current`. + +Rule (developer, November 13, 2026): a job node itself (`as_table`, `iter_data`, dataframe +methods, `flow`, `head`, the preview, `run`) is the CURRENT RUN view; `node.as_source()` is the +view of everything stored for the node. Before a run a node is empty; a run fills it, also when +everything is loaded. `run(load_only=True)` fills nodes from what is stored and computes nothing. +`node.run()` cascades to the job nodes it depends on (developer, later the same day). + +## 1. Measurements on the base (`2e9972e0`) + +Scratch scripts: `measure.py`, `replay.py`, `noderun.py` in the session scratchpad. +Pipeline: `DictSource(source_id="s", tags g,k)` -> `inc` (`rowwise`) -> `group_by("g")` +(`grouped`, operator) -> `total` (`summed`). Runs over 6 rows then 3 rows, one Delta store. + +| Read | sync | async | +|---|---|---| +| after run over 3 rows: rowwise / grouped / summed `as_table` | 3 / 2 / 2 | 3 / 2 / 2 | +| `as_source()` of rowwise / summed | 6 / 4 | 6 / 4 | +| fresh job (3 source rows), NOT run: rowwise / grouped(OFF) / summed `as_table` | **6** / 0 / **4** | same | +| fresh job, not run: `head()` rowwise | `rows`, 6 of 6 (stored) | same | +| fresh job: `summed.run()` alone (upstream OFF operator not run) | 0 rows, nothing computed | - | +| fresh job: source node `as_table` | 3 (bound source) | same | + +Operator node `grouped` before a run (fresh job over a store with two earlier runs): + +| cache mode | `as_table` | `head` | `as_source` | +|---|---|---|---| +| OFF | 0 | not_computed "not run; cache mode OFF stores nothing" | 0 | +| LOG | 0 | not_computed "rows of earlier runs are stored but not replayed" | 4 (both runs) | +| REPLAY (after two LOG runs) | **4** (both runs) | rows 4 | 4 | + +REPLAY after a run over 3 rows (sync and async alike): `grouped` holds **4 groups** +`(0,[1,21]) (0,[1,21,41]) (1,[11]) (1,[11,31,51])` -- every logged run, not the current run -- +and `summed` downstream gets 4 rows. The operator cache (`odb_v1`) stores no run id or time, so +"the latest logged run" cannot be told apart without a new stored column. + +`node.run()` with an upstream job node that has not run (chain `a` -> `b`, store holds a 6-row run, +source now 3 rows): `b.run()` reads `a`'s STORED rows (6, not the current 3) and returns 6 rows, +0 function calls; over an empty store it returns 0 rows silently. `node.run()` never runs upstream. + +Loaded job (`PipelineJob.load`, in-memory source unbound): function nodes are `READ_ONLY` stubs +showing every stored row (rowwise 6, summed 4); the OFF operator raises `RuntimeError` from +`as_table` (head: unavailable); source node raises `UnboundSourceError`; `loaded.run()` raises. +Loaded `rowwise.as_source()` reads 6 rows. + +Load statuses: a built job's function nodes are always `FULL`. A loaded node without a pod is +`READ_ONLY` (store attached) or `UNAVAILABLE`. `CACHE_ONLY` and `READ_ONLY` with a +`DataFunctionProxy` arise only from `FunctionJobNode.from_descriptor(..., function_pod=...)`, +which no public path calls (`PipelineJob.load` passes no pod). + +Orchestrators: both pass rows between nodes in their own buffers/channels; no node read is +involved in `job.run()`. Only `node.run()` reads its upstream node. + +Experiment (reverted): making every un-run read empty fails 400 of 4,464 job-related tests in +21 files (list in section 5). + +## 2. Proposal + +See the ask message (copied in section 6 once answered). + +## 3. Implementation sketch (after approval) + +- `FunctionJobNode`: no-run read empty (all load statuses); `head` NOT_COMPUTED with note; + `execute`/`async_execute` take `load_only` (lookup `get_cached_results` only, skip misses, count + them, no `_process_data_internal`, no writes); `_last_run_skipped` count; run mode flag. +- `OperatorJobNode`: REPLAY before run empty; load-only: OFF/LOG run the operator, LOG writes + nothing; REPLAY replays. +- Orchestrators: `load_only` threaded through; side-effect node passes rows through uncalled. +- `PipelineJob.run/run_async(*, load_only=False)`; loaded job allowed with `load_only=True`. +- One internal "run these job nodes" function shared by `job.run` and the `node.run` cascade. +- `RunSummary`/`NodeRunSummary` new field(s). +- Tests first (public interface, both orchestrators, 3.11 for async). +- Docs, changelog fragment, design_issues entry, CLAUDE.md + .zed/rules. + +## 5. Tests and docs that change + +Failing under the experiment (count per file): test_node_head_sorting 162, +test_loaded_operator_cache 74, test_function_node_arrow_types 58, test_loaded_job_readback 29, +test_delta_arrow_type_roundtrip 25, test_node_head 9, test_loaded_logical_schemas 7, +test_provisional_hash_warning 6, test_function_pod_node_stream 5, test_stored_data_context 4, +test_function_node_empty_data 4, test_job_node_rows_after_run 3, test_tensor_job_load_back 2, +test_materialize_source_info 2, test_function_node_caching 2, test_cached_function_pod_job 2, +test_doc_samples (function-pods.md) 1, test_operator_node 1, test_operator_node_non_active 1, +test_function_node_iteration 1, test_readback_datagram_id 1, test_polars_dataframe_values 1. +Plus `tests/test_core/streams/test_run.py::test_every_stream_class_has_run_without_arguments` +(a `load_only` keyword on a node's `run`). Full names: scratchpad `hack_failed_tests.txt`. + +Docs: docs/concepts/streams.md (run table row "A job node", head table row "A job node", the +"stream that has all of its rows" paragraph, line ~262), docs/concepts/function-pods.md (339-341 +no-run read; "Saving and loading a job" 662-746 and its sample at 689-694, the only doc sample +that reads a node without a run), docs/concepts/sources.md 169-172; docstrings of +`PipelineJob.load`/`run`, `FunctionJobNode.head/as_table/iter_data/run/clear_cache/_cannot_compute_error`, +`OperatorJobNode.iter_data/as_table/head`, `ViewStatus.NOTHING_STORED`, `StreamBase.head`. + +### Appendix: tests failing under the empty-before-run experiment (parametrizations collapsed) + +``` +tests/test_core/function_pod/test_cached_function_pod_job.py::test_saved_job_equals_the_uncached_pods_and_loads +tests/test_core/function_pod/test_function_node_arrow_types.py::test_store_read_back_keeps_tag_type +tests/test_core/function_pod/test_function_node_caching.py::TestPhase1Phase2PipelineEntryId::test_all_existing_entry_ids_skipped_in_phase2 +tests/test_core/function_pod/test_function_node_caching.py::TestPhase1Phase2PipelineEntryId::test_phase1_yields_existing_results +tests/test_core/function_pod/test_function_pod_node_stream.py::TestFunctionNodeInactive::test_as_table_returns_cached_results_when_data_function_inactive +tests/test_core/function_pod/test_function_pod_node_stream.py::TestFunctionNodeInactive::test_inactive_filled_db_serves_cached_results +tests/test_core/function_pod/test_function_pod_node_stream.py::TestFunctionNodeStaleness::test_as_table_auto_detects_stale_and_repopulates +tests/test_core/function_pod/test_function_pod_node_stream.py::TestFunctionNodeStaleness::test_clear_cache_produces_same_results_on_re_iteration +tests/test_core/function_pod/test_function_pod_node_stream.py::TestIterDatasDbPhase::test_db_served_results_have_correct_row_count +tests/test_core/function_pod/test_polars_dataframe_values.py::TestPipelineStores::test_saved_job_loaded_back_reads_the_frames +tests/test_core/function_pod/test_readback_datagram_id.py::test_a_fresh_job_on_the_store_reads_the_stored_ids +tests/test_core/nodes/test_function_node_empty_data.py::TestAsyncProcessDataInternalEmptyData::test_async_empty_data_cache_hit_returns_cached_result +tests/test_core/nodes/test_function_node_empty_data.py::TestAsyncProcessDataInternalEmptyData::test_async_empty_data_cache_miss_raises +tests/test_core/nodes/test_function_node_empty_data.py::TestProcessDataInternalEmptyData::test_cache_hit_returns_cached_result +tests/test_core/nodes/test_function_node_empty_data.py::TestProcessDataInternalEmptyData::test_cache_miss_raises_ephemeral_result_missing_error +tests/test_core/nodes/test_function_node_iteration.py::TestIterDatasReadOnly::test_iter_after_db_populated_hot_loads_without_compute +tests/test_core/operators/test_materialize_source_info.py::TestStoreWrittenBeforeFix::test_rerun_after_pre_fix_store +tests/test_core/operators/test_materialize_source_info.py::TestStoreWrittenBeforeFix::test_v014_store_is_not_read +tests/test_core/operators/test_operator_node_non_active.py::TestCacheModeNonActive::test_iter_data_replay_mode_returns_db_contents +tests/test_core/operators/test_operator_node.py::TestOperatorNodeRunAndStorage::test_replay_from_cache +tests/test_databases/test_delta_arrow_type_roundtrip.py::test_a_fresh_job_reads_the_tag_type +tests/test_databases/test_delta_arrow_type_roundtrip.py::test_a_loaded_job_reads_the_tag_type +tests/test_databases/test_delta_arrow_type_roundtrip.py::test_a_rerun_on_the_same_store_writes_no_pipeline_rows +tests/test_databases/test_delta_arrow_type_roundtrip.py::test_an_operator_replay_reads_the_tag_type +tests/test_docs/test_doc_samples.py::test_doc_page_samples_run +tests/test_logical_types/test_provisional_hash_warning.py::test_reading_stored_results_of_the_type_warns_once +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_binding_a_store_makes_a_storeless_loaded_node_readable +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_datetime_output_feeding_an_operator_loads_with_its_type +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_file_without_the_expected_rows_section_reads_without_a_check +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_latest_readable_result_wins_across_both_stores +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_live_node_hot_load_reads_the_latest_recomputed_result +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_loaded_function_node_as_table_all_info_has_stream_columns +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_loaded_function_node_as_table_returns_stored_rows +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_loaded_function_node_iter_data_yields_stored_rows +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_loaded_node_with_ephemeral_results_does_not_crash +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_loaded_operator_node_raises_instead_of_returning_empty +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_logical_type_output_schema_fails_naming_the_column_and_saved_type +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_newer_unavailable_result_does_not_expose_old_persistent_result +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_other_context_records_do_not_satisfy_saved_expectation +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_path_output_feeding_join_still_allows_job_load +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_recomputed_entry_is_read_once_with_its_latest_result +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_saving_a_loaded_job_keeps_the_expected_rows +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_store_content_warning_names_a_delta_stores_location +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_store_with_only_another_runs_records_reads_none_with_a_warning +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_unreconstructable_file_source_is_a_warning_with_the_reason +tests/test_pipeline/loaded_job/test_loaded_logical_schemas.py::test_registered_logical_output_keeps_join_available +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_a_format_0_1_0_file_finds_its_join_table_and_never_shows_its_rows_as_none +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_a_loaded_run_with_an_empty_result_reads_as_empty +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_a_pipeline_loaded_and_bound_to_its_sources_reuses_the_operator_cache +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_a_store_holding_only_other_runs_rows_gives_an_empty_result +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_an_absent_cache_table_gives_an_empty_result +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_fewer_rows_than_expected_warn_once_and_read_what_the_store_holds +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_get_cached_output_of_a_loaded_node_never_caches_unreadable_rows_as_none +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_loaded_replay_operator_reads_the_rows_its_run_stored +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_matching_counts_and_zero_expected_zero_found_do_not_warn +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_more_rows_than_expected_warn_and_read_them_all +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_without_saved_nullability_the_operator_still_reads_its_runs_table +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_without_the_run_content_hash_an_absent_table_raises_too +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_without_the_run_content_hash_head_says_why_it_shows_no_rows +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_without_the_run_content_hash_the_loaded_operator_raises +tests/test_pipeline/test_job_node_rows_after_run.py::TestRaisedRun::test_raised_run_reads_as_no_run +tests/test_pipeline/test_node_head_sorting.py::test_a_function_nodes_head_is_the_prefix_of_as_table_in_any_completion_order +tests/test_pipeline/test_node_head_sorting.py::test_an_operator_nodes_head_is_the_prefix_of_as_table +tests/test_pipeline/test_node_head.py::test_a_loaded_job_shows_stored_rows_and_never_raises +tests/test_pipeline/test_node_head.py::test_a_new_job_over_a_filled_store_reads_stored_rows_without_computing +tests/test_pipeline/test_node_head.py::test_head_keeps_the_tag_arrow_type_of_the_input +tests/test_pipeline/test_node_head.py::test_nodes_before_a_run_show_no_rows_and_compute_nothing +tests/test_pipeline/test_node_head.py::test_operator_node_cache_modes +tests/test_pipeline/test_stored_data_context.py::TestRowsOfAnotherDataContext::test_function_node_reads_current_rows_only +tests/test_pipeline/test_stored_data_context.py::TestSideEffects::test_function_node_upstream_of_a_refused_sink_is_not_refused +tests/test_torch/test_tensor_job_load_back.py::test_a_loaded_job_reads_its_stored_tensors +``` diff --git a/superpowers/plans/2026-11-13-itl-1095-proposal.md b/superpowers/plans/2026-11-13-itl-1095-proposal.md new file mode 100644 index 000000000..4478eaed4 --- /dev/null +++ b/superpowers/plans/2026-11-13-itl-1095-proposal.md @@ -0,0 +1,196 @@ +# ITL-1095 proposal (sent to the coordinator for decisions) + +Measurements: `2026-11-13-itl-1095-node-current-run-view.md` section 1 (same folder). +Recommendations are marked **(rec)**. Decisions needed: D1 to D11. + +## A. The rule as built, per node kind + +| Node | Before a run in this process (every read) | After a run | +|---|---|---| +| `FunctionJobNode` (FULL; the only state a built job has) | zero rows, typed columns; `head`/preview status `not_computed`, note below | the run's rows (#378); a load-only run: the loaded rows | +| same, after a run that raised / `clear_cache()` / stale (upstream re-ran or was rebound) | empty, as before a run (today: every stored row) | - | +| loaded function node (`READ_ONLY` stub) | empty (today: every stored row) | after `loaded.run(load_only=True)`: loaded rows | +| `CACHE_ONLY`, `READ_ONLY` with a proxy pod (only via `from_descriptor(..., function_pod=...)`, no public path) | empty | D9 | +| `OperatorJobNode` OFF / LOG | empty (unchanged: already so) | run's output (unchanged) | +| `OperatorJobNode` REPLAY | empty (today: union of every logged run) | D1 | +| loaded operator node | empty, no raise (today OFF raises `RuntimeError`) | after a load-only run: its output on the loaded rows | +| `SideEffectJobNode` | #377 (not touched): reads deliver nothing and return zero rows when nothing is held. Already fits the rule. | #377 | +| `SourceJobNode` | the bound source's rows, before and after (a source is not computed); unbound raises as today. **Yes, unchanged.** | same | + +`node.as_source()` unchanged: every stored record of the node (current data context). + +Head/preview note before a run (D10) **(rec)**: status `not_computed`, header like +`FunctionJobNode[rowwise] — not run; run() fills it, run(load_only=True) loads stored results only; node.as_source() shows every stored result`. +The stored count is NOT available without reading rows (the database protocol has no count; today's +head reads the whole stored slice to show stored rows). Exception: a loaded job whose file records +`expected_rows` for the node: "the saved run stored N rows". **(rec)** no count otherwise, and +`ViewStatus.NOTHING_STORED` then has no producer: remove it (greenfield) **(rec)**, or keep. + +## B. Load-only run: `job.run(*, load_only=False)`, `job.run_async(*, load_only=False)`, `node.run(*, load_only=False)` + +Keyword-only, default False. + +- Sources are read. +- Function node: computes each input's entry id, looks up the node's OWN stored entries (pipeline + records under its content hash, the lookup a run does today), forwards only the rows it loaded, + skips and counts inputs without one. No user function call, no side effect, no write to result + or pipeline tables. An entry whose result was ephemeral and is gone counts as not stored. + - **D2**: an input with no pipeline record for this node but a matching result in the + function's result store (same function in another node, or a source renamed with the same + rows; a normal run "reuses" it and writes a pipeline record). (a) skip it, count it as not + stored **(rec)**: what the node shows is then always a subset of `as_source()`. (b) show it + without writing a record. +- Operator node, **D3**: (a) OFF and LOG run their operator on the rows that reach them (cheap, + not stored; LOG writes nothing in a load-only run), REPLAY replays as in a normal run + **(rec)**; (b) only cached operators produce rows: OFF (the only mode a job sets) forwards + nothing, so everything below any operator is empty: load-only becomes useless for + `group_by`, `join`, `select` pipelines. + - GroupBy hazard under (a): if the function node above lost inputs (not stored), the groups are + PARTIAL; a partial group's input differs from the full group's, so the function below misses + -- unless an earlier, smaller run stored exactly that partial group. Measured case: store has + the 3-row run, source now 6 rows, `rowwise` stored for 3 of 6 -> load-only groups 3 rows -> + `summed` HITS the 3-row run's totals (0,22),(1,11) and shows them as this run's. Options: + (i) the summary marks every node downstream of a node with skipped inputs as "inputs + incomplete (rowwise skipped 3)" and the docs say so **(rec for the tag)**; (ii) a node below + a node with skipped inputs forwards nothing (strict; also blanks row-wise chains, which are + exact); (iii) nothing. +- Side-effect node (D4): **(rec)** the orchestrator passes its input rows through without calling + it (no delivery, no log write; no change to `side_effects.py`, which #377 owns); the node then + holds nothing and reads empty. After #377 merges it could hold the passed rows instead. + A ctx_arg function node is a function node: loads, calls nothing. +- Both orchestrators: same per-node behaviour; `load_only` passed through `run`/`run_async` to + `execute`/`async_execute`. +- (d) `error_policy`: nothing to apply it to (no function runs); store read errors and + `TypeIdentityError` raise as in any run, operator errors raise as today. Observers: + `on_run_start/end`, `on_node_start/end` per node; a loaded row gets `on_data_start` + + `on_data_end(cached=True)`; a skipped input fires no row hook (no observer protocol change; + the node counts it and the job reads the count). A user-supplied status/logging observer writes + its own observability rows as in any run (not result/pipeline tables) -- say if load-only should + suppress them. +- **D5 RunSummary shape (rec)**: `NodeRunSummary` gains `not_stored: int = 0` (inputs skipped + because no result is stored, load-only only) and `inputs_incomplete: tuple[str, ...] = ()` + (upstream nodes that skipped inputs, D3(i)); loaded rows count in the existing `reused` ("read + from the store"), `computed` and `failed` are 0; `rows` = computed + reused + failed + + not_stored; `RunSummary.load_only: bool`. Repr line: + `rowwise: 6 rows: 3 loaded, 3 not stored (skipped)` and + `summed: 2 rows: 2 loaded (inputs incomplete: rowwise)`. Alternative: a separate `loaded` field. + Summary nodes stay function nodes only, as today. +- (f) Hashes: none move (no identity, path or record-id change). A load-only run writes nothing: + tests snapshot every table path and row count of the store before and after, both + orchestrators, including the operator cache in LOG and a side-effect log. + +## C. `node.run()` cascade (developer's rule) + +- Today (measured): `node.run()` never runs upstream. With an un-run upstream function node it + reads the upstream's STORED rows (6, though the source has 3 now) and processes them; over an + empty store, or behind an un-run OFF operator, it processes nothing and returns 0 rows silently. +- **(rec)** `node.run(*, load_only=False)` runs the sub-graph of the job nodes it depends on plus + itself, in dependency order, each once (diamonds), with ONE mechanism shared with `job.run`: an + internal function that takes a node DAG and runs it on the `SyncPipelineOrchestrator` + (`job.run` passes its whole DAG; `node.run` the ancestor DAG found by walking `upstreams`, + which in a job are the job nodes). Same rows, stored records, error_policy ("continue") and + per-node counts as `job.run` for that sub-graph. Sources are read, not run. +- **D7 "not necessarily trigger upstreams"** **(rec)**: an ancestor that holds a CURRENT run is + not run again; its held rows feed its successors. Current = it holds a completed run in this + process and `is_stale` is False (the existing judgement: own `last_modified` set and no upstream + or producer modified later; a re-run or rebound upstream makes it stale) and, for a full + cascade, that run was not load-only. So after a load-only fill, a full `node.run()` re-runs + every load-only-filled ancestor in full first (computes their missing entries) **(rec, + matches your expectation)**; option (b): use the loaded rows as they are. A load-only cascade + accepts a fully-run ancestor as current (its rows are all stored). +- Side-effect ancestors: a full cascade delivers exactly as `job.run` (with `track_completion` + only the undelivered); a load-only cascade delivers nothing (passes rows through). An ancestor + side-effect node that holds a current run (#377) is not run again, so nothing is re-delivered. +- Counts: `node.run()` still returns the node's `StreamView`. **(rec)** the cascade's + `RunSummary` is kept on the node as `node.last_run` (mirrors `job.last_run`), and a cascade + with failed rows issues the same `RowFailuresWarning` as `job.run`. Option: attach it to the + returned view (`view.run_summary`). +- Commits: as a node run on its own does today (every row) plus a flush at the end. Option: the + job's `commit_every_rows=100` cadence. +- **D6 conflict with a standing rule**: CLAUDE.md (ITL-1081, developer) "Every stream, source and + node has `run()` with no arguments ... Never give `run()` arguments", enforced by + `tests/test_core/streams/test_run.py::test_every_stream_class_has_run_without_arguments`. + **(rec)** amend to: "`run()` takes no positional arguments; a job node's `run()` takes one + keyword-only argument, `load_only`" and narrow the test to that. Option: `load_only` on + `job.run` only, and `node.run_load_only()`... (not recommended). + +## D. Loaded jobs (`PipelineJob.load`) + +- Every loaded node is empty before a run (function stubs and operators alike; operators no + longer raise from a read). `node.as_source()` reads everything stored (measured: loaded + `rowwise.as_source()` = 6 rows), with or without sources. +- `loaded.run(load_only=True)` **(rec)**: allowed. Sources come from the file (CSV, Delta) or + `loaded.bind(sources=...)` (in-memory sources are not saved). Nodes below an unbound source are + excluded and stay empty, as in any partial run. A function stub can do a load-only run: entry + ids need no function pod (system tags + input data hash), and the stub reads through its stored + paths and the saved run's content hash. Operators are rebuilt from the blueprint and run. + Without any live source there are no inputs to look up: a load-only run is impossible below + that source, and `as_source()` is the only way in. +- `loaded.run()` without `load_only`, and a loaded `node.run()`: raise as today, message pointing + to `run(load_only=True)` and `as_source()`. Loaded `node.run(load_only=True)`: the cascade. +- D8: confirm (rec) above. Risk: the stub's lookups use the saved run's content hash; a bound + source with another `source_id` finds nothing (all inputs counted not stored). + +## E. Other load statuses (D9) + +`CACHE_ONLY` (no upstream) and `READ_ONLY` with a proxy pod are reachable only through +`FunctionJobNode.from_descriptor(..., function_pod=...)` (tests only). Today their `run()` returns +every stored row. **(rec)** both empty before a run; `READ_ONLY` proxy: `run(load_only=True)` +loads, `run()` raises (cannot compute); `CACHE_ONLY`: no input, so `run(load_only=True)` raises +pointing to `as_source()`. (Candidates for removal later; out of scope.) + +## F. REPLAY (D1, not decided by me) + +Measured: after a run over 3 rows a REPLAY node holds the union of every LOG run (4 groups, two +from a 6-row run) and feeds it downstream. The rule forces only the BEFORE-a-run part (empty). +Options: (R1) REPLAY after a run keeps meaning "the stored output instead of computing": the +union of every logged run under its content hash; only the before-run read changes **(minimal +for the tag)**. (R2) REPLAY replays only the latest logged run: `odb_v1` stores no run id or time, +so this needs a new stored column -> layout change (`odb_v2`), not for the tag. (R3) REPLAY +replays only the logged rows of THIS run's inputs: operator outputs have no input entry ids, so +not possible without a stored-schema change either. + +## G. Tests and docs that change (e) + +Experiment (every no-run read empty, reverted): 400 of 4,464 job-related tests fail in 21 files: +test_node_head_sorting 162, test_loaded_operator_cache 74, test_function_node_arrow_types 58, +test_loaded_job_readback 29, test_delta_arrow_type_roundtrip 25, test_node_head 9, +test_loaded_logical_schemas 7, test_provisional_hash_warning 6, test_function_pod_node_stream 5, +test_stored_data_context 4, test_function_node_empty_data 4, test_job_node_rows_after_run 3 +(#378's `test_raised_run_reads_as_no_run`), test_tensor_job_load_back 2, +test_materialize_source_info 2, test_function_node_caching 2, test_cached_function_pod_job 2, +test_operator_node 1, test_operator_node_non_active 1, test_function_node_iteration 1, +test_readback_datagram_id 1, test_polars_dataframe_values 1, test_doc_samples +(function-pods.md) 1; plus test_run.py's no-argument test (D6) and the loaded-job "cannot run" +tests (test_loaded_job_still_cannot_run, ..._operator_only_job_cannot_run, +..._without_a_store_cannot_run, test_loaded_function_node_cannot_execute message). Each is +changed to read after `run()`/`run(load_only=True)`, or to `as_source()` where it is about the +stored records. Workshop scenario tests: unaffected (they run first, or read the store directly). +Full list: plan file appendix. + +Docs: docs/concepts/streams.md (run table and head table rows for "A job node", the "stream that +has all of its rows" paragraph, ~262), docs/concepts/function-pods.md (339-341 no-run read; +"Saving and loading a job" 662-746 and its sample 689-694, the only doc sample that reads a node +without a run: becomes bind + `loaded.run(load_only=True)`, or `as_source()`; a new "Loading +without computing" subsection), docs/concepts/sources.md 169-172; docstrings of +`PipelineJob.load/run/run_async`, node `head/as_table/iter_data/run/clear_cache`, operator +`iter_data/as_table/head`, `ViewStatus`, `StreamBase.head`. Changelog fragment `ITL-1095.md` +(Breaking: a node read before a run is empty, loaded nodes too, `node.run()` cascades; Added: +`load_only`, summary fields). CLAUDE.md/.zed/rules: replaces #378's sentence, amends the run() +rule (D6). + +## H. PR #346 (g) + +Only `tests/test_core/alignment/test_nodes.py::read_back(how="load")` reads a loaded job +without a run: `test_rows_read_back_from_the_store_are_aligned` (216 parametrizations: loaded +`scaled`/`joined` `as_table`/`head`) and `test_stored_records_read_back_are_aligned` (54; reads +`iter_data`). It will need `loaded.bind(sources=...)` (its sources are in-memory) and +`loaded.run(load_only=True)` before `return loaded`; `joined` is attached in REPLAY (D1 decides +what it then holds). `test_job.py` and the rerun modes read after a run: unaffected. + +## Sizing + +Large for Sunday: node rule + load-only through both orchestrators + summary + cascade + loaded +jobs + ~400 test updates + docs. If something must be cut for the tag, I would drop first D's +loaded-job load-only run (loaded reads empty, `as_source()` the way in, `loaded.run()` raises), +then C's "current ancestor" skip (a cascade always runs every ancestor). From 1d990178f715dcf6aa264f1015ab8106585c99b7 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:23:53 -0800 Subject: [PATCH 04/63] test(nodes): failing tests: a job node reads empty and stale before a run (ITL-1095) Co-Authored-By: Claude Opus 5.5 --- .../test_job_node_current_run_view.py | 208 ++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 tests/test_pipeline/test_job_node_current_run_view.py diff --git a/tests/test_pipeline/test_job_node_current_run_view.py b/tests/test_pipeline/test_job_node_current_run_view.py new file mode 100644 index 000000000..5b6cf24cd --- /dev/null +++ b/tests/test_pipeline/test_job_node_current_run_view.py @@ -0,0 +1,208 @@ +"""A job node shows the current run; ``node.as_source()`` shows what is stored (ITL-1095). + +Developer, November 13, 2026: a job node itself (``as_table``, ``iter_data``, the dataframe +methods, ``flow``, ``head``, the preview, ``run``) is the current run view; ``as_source()`` +is the view of everything stored for the node. Before a run in this process a node is empty +and stale, whatever its store holds; a run fills it. + +One store holds the results of two earlier runs (6 source rows, then 3). A fresh job over +that store, and a job loaded from a saved file, read empty from every node until they run. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.core.streams.view import ViewStatus +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.sources import DictSource +from orcapod.types import CacheMode + +CALLS: list[str] = [] + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + """Return ``x + 1``.""" + CALLS.append("inc") + return x + 1 + + +@function_pod(output_keys="total") +def total(y: list[int]) -> int: + """Return the sum of a group's values.""" + CALLS.append("total") + return sum(y) + + +ROWS = [{"g": i % 2, "k": i, "x": 10 * i} for i in range(6)] +ORCHESTRATORS = {"sync": SyncPipelineOrchestrator, "async": AsyncPipelineOrchestrator} +JOB_NODES = ("rowwise", "grouped", "summed") + + +@pytest.fixture(autouse=True) +def _reset_calls() -> None: + CALLS.clear() + + +def _build( + data: list[dict[str, int]], + db: Any, + cache_mode: CacheMode | None = None, +) -> PipelineJob: + """``rowwise`` (function) -> ``grouped`` (GroupBy operator) -> ``summed`` (function).""" + src = DictSource(data, tag_columns=["g", "k"], source_id="s") + job = PipelineJob(name="j", database=db) + with job: + rowwise = inc.pod(src, label="rowwise") + grouped = rowwise.group_by("g", label="grouped") + total.pod(grouped, label="summed") + if cache_mode is not None: + job.nodes["grouped"].attach_databases( + pipeline_database=db.at("j"), cache_mode=cache_mode + ) + return job + + +def _populated_store( + orchestrator: str = "sync", cache_mode: CacheMode | None = None +) -> InMemoryArrowDatabase: + """A store holding the results of a run over 6 rows and a run over 3 rows.""" + db = InMemoryArrowDatabase() + for data in (ROWS, ROWS[:3]): + _build(data, db, cache_mode).run(orchestrator=ORCHESTRATORS[orchestrator]()) + CALLS.clear() + return db + + +def _assert_empty(node: Any, columns: list[str]) -> None: + """Every read of ``node`` returns zero rows with the node's columns.""" + table = node.as_table() + assert table.num_rows == 0 + assert table.column_names == columns + assert list(node.iter_data()) == [] + assert node.as_polars_df().height == 0 + assert node.head().table.num_rows == 0 + assert node.head().table.column_names == columns + + +def _assert_points_to_run_and_stored_view(node: Any) -> None: + """The head of a node that has not run says so and names both ways in.""" + head = node.head() + assert head.status is ViewStatus.NOT_COMPUTED + header = head.header() + assert "not run" in header + assert "run()" in header + assert "as_source()" in header + assert "not run" in node._repr_html_() + + +class TestFreshJobOverAPopulatedStore: + """A job built over a store that holds results reads empty until it runs.""" + + @pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) + def test_every_read_of_every_node_is_empty_before_a_run( + self, orchestrator: str + ) -> None: + db = _populated_store(orchestrator) + job = _build(ROWS[:3], db) + expected_columns = { + label: _build(ROWS[:3], db).nodes[label].as_table().column_names + for label in JOB_NODES + } + for label in JOB_NODES: + node = job.nodes[label] + _assert_empty(node, expected_columns[label]) + _assert_points_to_run_and_stored_view(node) + assert CALLS == [] + + def test_the_stored_view_shows_every_stored_record(self) -> None: + db = _populated_store() + job = _build(ROWS[:3], db) + # Both earlier runs' rows: 6 rowwise results, 2 + 2 group totals. + assert job.nodes["rowwise"].as_source().as_table().num_rows == 6 + assert job.nodes["summed"].as_source().as_table().num_rows == 4 + assert job.nodes["rowwise"].as_table().num_rows == 0 + + def test_every_node_of_a_fresh_job_is_stale(self) -> None: + job = _build(ROWS[:3], _populated_store()) + for label in JOB_NODES: + assert job.nodes[label].is_stale, label + + @pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) + def test_a_run_fills_the_nodes_with_this_runs_rows(self, orchestrator: str) -> None: + db = _populated_store(orchestrator) + job = _build(ROWS[:3], db) + job.run(orchestrator=ORCHESTRATORS[orchestrator]()) + assert CALLS == [] # every result was stored + assert job.nodes["rowwise"].as_table().num_rows == 3 + assert sorted( + (r["g"], r["total"]) for r in job.nodes["summed"].as_table().to_pylist() + ) == [(0, 22), (1, 11)] + for label in JOB_NODES: + assert not job.nodes[label].is_stale, label + assert job.nodes[label].head().status is ViewStatus.ROWS, label + + +@pytest.mark.parametrize("mode", [CacheMode.OFF, CacheMode.LOG, CacheMode.REPLAY]) +def test_an_operator_node_reads_empty_before_a_run_in_every_cache_mode( + mode: CacheMode, +) -> None: + """REPLAY included: before a run it no longer shows every logged run.""" + # LOG runs fill the operator cache, so REPLAY has stored rows to (not) show. + db = _populated_store(cache_mode=CacheMode.LOG) + job = _build(ROWS[:3], db, cache_mode=mode) + node = job.nodes["grouped"] + _assert_empty(node, ["g", "y"]) + _assert_points_to_run_and_stored_view(node) + assert node.is_stale + + +class TestAfterARun: + """What empties a node that held a run: clearing it, or a change upstream.""" + + @pytest.mark.parametrize("label", list(JOB_NODES)) + def test_clear_cache_empties_the_node_and_makes_it_stale(self, label: str) -> None: + job = _build(ROWS[:3], InMemoryArrowDatabase()) + job.run() + node = job.nodes[label] + columns = node.as_table().column_names + node.clear_cache() + assert node.is_stale + _assert_empty(node, columns) + + def test_a_rebound_source_empties_its_dependent_nodes(self) -> None: + job = _build(ROWS[:3], InMemoryArrowDatabase()) + job.run() + source_name = next(iter(job.sources)) # the one source slot + job.bind( + sources={ + source_name: DictSource(ROWS, tag_columns=["g", "k"], source_id="s") + } + ) + for label in JOB_NODES: + node = job.nodes[label] + assert node.is_stale, label + assert node.as_table().num_rows == 0, label + assert node.head().status is ViewStatus.NOT_COMPUTED, label + + +def test_a_loaded_job_reads_empty_until_it_runs(tmp_path: Path) -> None: + """A loaded job's nodes are empty and do not raise; ``as_source()`` reads the store.""" + db = InMemoryArrowDatabase() + job = _build(ROWS, db) + job.run() + job.save(tmp_path / "job.json") + loaded = PipelineJob.load(tmp_path / "job.json", database=db) + for label in JOB_NODES: + node = loaded.nodes[label] + assert node.as_table().num_rows == 0, label + assert list(node.iter_data()) == [], label + assert node.head().status is ViewStatus.NOT_COMPUTED, label + assert "not run" in node.head().header(), label + assert loaded.nodes["rowwise"].as_source().as_table().num_rows == 6 From b1a8d2174865286621b480755c1c836aeb356841 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:27:02 -0800 Subject: [PATCH 05/63] feat(nodes)!: a job node reads empty and stale before a run (ITL-1095) A function or operator job node shows only the rows of its current run in this process: before a run, after clear_cache(), after a run that raised, and once stale (any job node it depends on, or a rebound source, changed since) every read returns zero typed rows and head() points to run() and node.as_source(). REPLAY no longer shows stored rows before a run; a loaded node reads empty instead of every stored row (function) or raising (operator). Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/function_node.py | 188 ++++++------------ src/orcapod/core/nodes/operator_node.py | 165 +++++++-------- src/orcapod/core/nodes/source_node.py | 27 ++- src/orcapod/core/nodes/staleness.py | 34 ++++ src/orcapod/core/streams/view.py | 7 +- .../test_job_node_rows_after_run.py | 23 +-- 6 files changed, 207 insertions(+), 237 deletions(-) create mode 100644 src/orcapod/core/nodes/staleness.py diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index aa9c50dff..7b28f3b4d 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -36,10 +36,13 @@ ) from orcapod.core.streams.view import ( BLUEPRINT_NOTE, + JOB_NODE_NOT_RUN_NOTE, + STALE_NOTE, ViewStatus, StreamView, ) from orcapod.core.tracker import DEFAULT_TRACKER_MANAGER +from orcapod.core.nodes.staleness import upstream_job_node_stale from orcapod.core.datagrams.tag_data import Data, EmptyData, Tag from orcapod.errors import ( EphemeralResultMissingError, @@ -1117,6 +1120,8 @@ def __init__( result_database=result_database, ephemeral_database=ephemeral_database, ) + # A node that has not run in this process holds nothing and is stale (ITL-1095). + self._set_modified_time(None) # ------------------------------------------------------------------ # node_config property @@ -1513,14 +1518,27 @@ def from_descriptor( def clear_cache(self) -> None: """Clear in-memory output caches, content hash cache, and node identity path cache. - The node then holds no run: it reads as a node with no completed run in this - process (every row stored under its content hash). + The node then holds no run: it is stale and reads empty, as before its first run + in this process (ITL-1095). Its stored results stay stored (``as_source()``). """ super().clear_cache() self._cached_output_datas.clear() self._cached_output_table = None self._cached_content_hash_column = None self._run_entry_counts = None + # No content established: the node, and every node downstream, is stale. + self._set_modified_time(None) + + @property + def is_stale(self) -> bool: + """Whether the rows this node holds are out of date, or it holds none. + + True before the node's first run in this process and after ``clear_cache()`` + (no modified time), when an input or the function pod was modified after the node's + run (``StreamBase.is_stale``), or when any job node it depends on is stale + (ITL-1095). A stale node reads empty until it runs again. + """ + return super().is_stale or upstream_job_node_stale(self.upstreams) def _held_pairs(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: """Yield the (tag, data) rows this node holds; every read of them goes through here. @@ -1529,17 +1547,18 @@ def _held_pairs(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: (two equal inputs give two rows, as downstream received them). The in-memory cache may hold more (results of other inputs, read from the store), which stay cached for reuse but are not rows of this run (ITL-1093). Without a completed - run in this process, every held row. Rows come in the order the cache holds - them. An input without output (an inactive function) is not a row. + run in this process there are no rows, whatever the cache or the store holds + (ITL-1095). Rows come in the order the cache holds them. An input without output + (an inactive function) is not a row. """ - held = self._cached_output_datas counts = self._run_entry_counts if counts is None: - pairs: Iterable[tuple[TagProtocol, DataProtocol | None]] = held.values() - else: - pairs = ( - pair for eid, pair in held.items() for _ in range(counts.get(eid, 0)) - ) + return + pairs: Iterable[tuple[TagProtocol, DataProtocol | None]] = ( + pair + for eid, pair in self._cached_output_datas.items() + for _ in range(counts.get(eid, 0)) + ) for tag, data in pairs: if data is not None: yield tag, data @@ -3136,18 +3155,15 @@ def head( columns: ColumnConfig | dict[str, Any] | None = None, all_info: bool = False, ) -> StreamView: - """Return the first rows the node holds or has stored; never runs the function. + """Return the first rows of the node's current run; never runs the function. After a run (or ``execute``) the rows are the node's results in memory for that - run's inputs, the first of what ``as_table`` returns. Otherwise they are the first of its stored - rows: the node's whole stored slice is read and nothing is kept on the node. - With nothing stored, there are no rows. The node has all of its rows, so with - ``sort_by_tags`` (or ``all_info``) the head is exactly - ``as_table(...).slice(0, n)``, whatever order the rows were computed in. See - ``StreamBase.head``. - - Warns: - StoreContentWarning: As for ``as_table``. + run's inputs, the first of what ``as_table`` returns. A node that has not run in + this process, or whose input changed since (stale), shows no rows and says so, + pointing to ``run()`` and to ``as_source()`` for its stored results (ITL-1095); + the store is not read. The node has all of its rows, so with ``sort_by_tags`` + (or ``all_info``) the head is exactly ``as_table(...).slice(0, n)``, whatever + order the rows were computed in. See ``StreamBase.head``. """ from orcapod.pipeline.serialization import LoadStatus @@ -3158,51 +3174,15 @@ def head( return self._head_without_rows( config, ViewStatus.UNAVAILABLE, note="no store attached" ) - current = status == LoadStatus.CACHE_ONLY or not self.is_stale - if current and ( - self._run_entry_counts is not None or self._cached_output_datas - ): - return self._head_of_pairs( - list(self._held_pairs()), n, config, origin="held" + if self._run_entry_counts is None: + return self._head_without_rows( + config, ViewStatus.NOT_COMPUTED, note=JOB_NODE_NOT_RUN_NOTE ) - return self._head_from_store(n, config) - - def _head_from_store(self, n: int, config: ColumnConfig) -> StreamView: - """Return the first ``n`` of this node's stored rows. - - Reads the node's whole stored slice (pipeline table joined with the result - table; no bounded read exists yet) and keeps nothing on the node. - """ - if self._function_pod is None: - # A loaded node: its table is built from the stored Arrow records. - self._check_expected_rows() - fetched = self._fetch_joined_records() - stored = None if fetched is None else self._stored_rows_stream(fetched) - stored_ids = set(stored[0]) if stored is not None else set() - missing = 0 - if fetched is not None: - missing = sum( - 1 for k in fetched.empty_data_tokens if k not in stored_ids - ) - note = ( - f"{missing:,} {'result' if missing == 1 else 'results'} not shown: kept " - "in an ephemeral store, which is not saved with the job" - if missing - else None + if status != LoadStatus.CACHE_ONLY and self.is_stale: + return self._head_without_rows( + config, ViewStatus.NOT_COMPUTED, note=STALE_NOTE ) - if stored is not None: - return ( - stored[1] - .head(n, columns=config) - .for_stream(self, origin="stored", note=note) - ) - return self._head_without_rows(config, ViewStatus.NOTHING_STORED, note=note) - if self._cached_function_pod is None: - return self._head_without_rows(config, ViewStatus.NOT_COMPUTED) - entries = list(self._load_cached_entries().values()) - if not any(data is not None for _, data in entries): - return self._head_without_rows(config, ViewStatus.NOTHING_STORED) - return self._head_of_pairs(entries, n, config, origin="stored") + return self._head_of_pairs(list(self._held_pairs()), n, config, origin="held") # ------------------------------------------------------------------ # Iteration @@ -3214,54 +3194,33 @@ def as_table( columns: ColumnConfig | dict[str, Any] | None = None, all_info: bool = False, ) -> "pa.Table": - """Return this node's results as a PyArrow table. + """Return the rows of this node's current run as a PyArrow table. - A node loaded by ``PipelineJob.load`` has no function pod. Its table is - built from the Arrow records its saved run stored, so every column keeps - its stored type. Row order follows the store. With nothing stored, the - table has zero rows and the node's columns. + Read-only: never computes and never reads the store. Before a run in this + process, or once the node is stale, the table has zero rows and the node's + columns (ITL-1095); ``as_source()`` reads what the store holds. Raises: - RuntimeError: If the node was loaded and no store is attached, or some of - its results were kept in an ephemeral store, which is not saved with - the job. - - Warns: - StoreContentWarning: Once per store, if the node was loaded from a job file - that records another number of pipeline records for its run than the - store holds. The read returns what the store holds. + RuntimeError: If the node was loaded and no store is attached. """ - if self._function_pod is None: - self._require_readable_stub() - fetched = self._fetch_joined_records() - if fetched is None: - return super().as_table(columns=columns, all_info=all_info) - stored = self._stored_rows_stream(fetched) - stored_ids = set(stored[0]) if stored is not None else set() - ephemeral = [k for k in fetched.empty_data_tokens if k not in stored_ids] - if ephemeral: - raise RuntimeError( - f"FunctionJobNode {self.label!r}: the saved run kept " - f"{len(ephemeral)} of its results in an ephemeral store, which " - "is not saved with the job, so they cannot be read into a " - "table. iter_data() yields them as EmptyData placeholders." - ) - if stored is not None: - return stored[1].as_table(columns=columns, all_info=all_info) + from orcapod.pipeline.serialization import LoadStatus + + if self.load_status == LoadStatus.UNAVAILABLE: + raise self._no_store_error() return super().as_table(columns=columns, all_info=all_info) def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: - """Yield this node's (tag, data) rows. + """Yield the (tag, data) rows of this node's current run. - Strictly read-only — never triggers computation. Callers must call - ``run()`` or ``execute()`` first if they want results computed. + Strictly read-only: never computes and never reads the store. Callers must + call ``run()`` (or run the job) first. After a completed run (``execute``, ``async_execute``, ``run()`` or a job run) the rows are exactly those of that run's inputs, on either orchestrator, even - though results of other inputs stay cached and stored (ITL-1093). A run that - raised defines no rows. Without a completed run in this process, on the first - call with an empty in-memory store and a DB attached, hot-loads every record - stored under the node's content hash (one-shot, no recompute) and yields them. + though results of other inputs stay cached and stored (ITL-1093). A node that + has not run in this process, whose run raised, or that is stale (its input + changed since its run) yields nothing (ITL-1095); ``as_source()`` reads every + row stored for it. Raises: RuntimeError: If ``load_status`` is UNAVAILABLE. @@ -3271,34 +3230,9 @@ def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: status = self.load_status if status == LoadStatus.UNAVAILABLE: raise self._no_store_error() - - if status == LoadStatus.CACHE_ONLY: - # Upstream unavailable; serve entirely from DB. - if not self._cached_output_datas: - loaded = self._load_cached_entries() - self._cached_output_datas.update(loaded) - if loaded: - self._cached_output_table = None - self._cached_content_hash_column = None - yield from self._held_pairs() - return - - # FULL / READ_ONLY — in-memory store may be populated from computation - # (via execute/run) or hot-loaded from DB. - if self.is_stale: + # A CACHE_ONLY node has no live input to be stale against. + if status != LoadStatus.CACHE_ONLY and self.is_stale: self.clear_cache() - - if self._function_pod is None: - # A loaded node: its rows can come only from the store. - self._check_expected_rows() - if not self._cached_output_datas and self._cached_function_pod is not None: - # Hot-load from DB on the first call when store is empty. - loaded = self._load_cached_entries() - self._cached_output_datas.update(loaded) - if loaded: - self._cached_output_table = None - self._cached_content_hash_column = None - yield from self._held_pairs() def run(self) -> StreamView: diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index bf2517c92..edb9e0b73 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -32,6 +32,7 @@ check_head_size, ) from orcapod.core.streams.view import ( + JOB_NODE_NOT_RUN_NOTE, STALE_NOTE, BLUEPRINT_NOTE, ViewStatus, @@ -39,6 +40,7 @@ head_of, ) from orcapod.core.tracker import DEFAULT_TRACKER_MANAGER +from orcapod.core.nodes.staleness import upstream_job_node_stale from orcapod.errors import ( PipelineJobRequiredError, StoreReadError, @@ -710,6 +712,8 @@ def __init__( pipeline_database=pipeline_database, cache_mode=cache_mode, ) + # A node that has not run in this process holds nothing and is stale (ITL-1095). + self._set_modified_time(None) # ------------------------------------------------------------------ # as_node — return the lightweight OperatorNode equivalent @@ -739,10 +743,16 @@ def set_ephemeral_store(self, store: "ArrowDatabaseProtocol | None") -> None: # ------------------------------------------------------------------ def clear_cache(self) -> None: - """Clear output caches and node identity path cache.""" + """Clear output caches and node identity path cache. + + The node then holds no run: it is stale and reads empty, as before its first run + in this process (ITL-1095). Rows it stored stay stored (``as_source()``). + """ super().clear_cache() # clears _node_identity_path_cache + _update_modified_time self._cached_output_stream = None self._cached_output_table = None + # No content established: the node, and every node downstream, is stale. + self._set_modified_time(None) # ------------------------------------------------------------------ # attach_databases @@ -1424,7 +1434,10 @@ def run(self) -> StreamView: ) self._update_modified_time() computed = True - table = self.as_table() + # Read the kept output itself: a lazy input that finishes computing during this + # read stamps itself after the node, which would make as_table() judge it stale. + assert self._cached_output_stream is not None + table = self._cached_output_stream.as_table() if computed: # Without a database and in OFF mode the output stream is lazy: as_table() # computes it, and an uncomputed input finishes only then. Stamp the node @@ -1437,26 +1450,41 @@ def run(self) -> StreamView: origin = "computed" if kept is None else head_of(kept.head(1), self).origin return self._view_of_rows(table, origin) - def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: - """Return an iterator over (tag, data) pairs. + @property + def is_stale(self) -> bool: + """Whether the rows this node holds are out of date, or it holds none. + + True before the node's first run in this process and after ``clear_cache()`` + (no modified time), when an input or the operator was modified after the node's + run (``StreamBase.is_stale``), or when any job node it depends on is stale + (ITL-1095). A stale node reads empty until it runs again. + """ + return super().is_stale or upstream_job_node_stale(self.upstreams) - Read-only: never triggers computation. Yields the rows of the latest - ``run()``, ``execute()`` or ``async_execute()``. With nothing cached in - memory (before a run, or after ``clear_cache()``), a node in - ``CacheMode.REPLAY`` with a database yields the rows stored there; in - any other mode it yields nothing. Call ``node.is_stale`` before - iterating if you need to detect outdated cached data. + def _current_output(self) -> StreamProtocol | None: + """The output of the node's current run, or ``None`` when it holds none. - Loaded nodes raise when no readable stored cache is available. + A node holds a run's output from the end of ``run()``, ``execute()`` or + ``async_execute()`` until it is cleared or becomes stale (an input or the + operator changed since). Before a run in this process it holds none, whatever + its cache mode and store hold (ITL-1095). """ - if self._cached_output_stream is not None: - return self._cached_output_stream.iter_data() - db_stream = self._load_cached_stream_from_db() - if db_stream is not None: - return db_stream.iter_data() - if self._loaded: - raise self._no_stored_results_error() - return iter([]) + if self._cached_output_stream is None or self.is_stale: + return None + return self._cached_output_stream + + def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: + """Return an iterator over the (tag, data) rows of the node's current run. + + Read-only: never computes and never reads the store. Yields the rows of the + latest ``run()``, ``execute()`` or ``async_execute()``; nothing before a run in + this process, after ``clear_cache()`` or once the node is stale, in every cache + mode (ITL-1095). ``as_source()`` reads what the node stored (``LOG``). + """ + current = self._current_output() + if current is None: + return iter([]) + return current.iter_data() def as_table( self, @@ -1464,34 +1492,24 @@ def as_table( columns: ColumnConfig | dict[str, Any] | None = None, all_info: bool = False, ) -> "pa.Table": - """Return the output as a PyArrow table. + """Return the output of the node's current run as a PyArrow table. - Read-only: never triggers computation. Returns the rows of the latest - ``run()``, ``execute()`` or ``async_execute()``. With nothing cached in - memory (before a run, or after ``clear_cache()``), a node in - ``CacheMode.REPLAY`` with a database returns the rows stored there; in - any other mode it returns a zero-row table with the node's columns. - - Loaded nodes raise when no readable stored cache is available. + Read-only: never computes and never reads the store. Returns the rows of the + latest ``run()``, ``execute()`` or ``async_execute()``; before a run in this + process, after ``clear_cache()`` or once the node is stale, a zero-row table + with the node's columns, in every cache mode (ITL-1095). """ - if self._cached_output_stream is not None: - return self._cached_output_stream.as_table( - columns=columns, all_info=all_info - ) - db_stream = self._load_cached_stream_from_db() - if db_stream is not None: - return db_stream.as_table(columns=columns, all_info=all_info) - if self._loaded: - raise self._no_stored_results_error() - # No cached or stored records yet: construct a zero-row table with the - # correct schema and route it through ArrowTableStream so that the - # ColumnConfig / all_info shaping is applied consistently. + current = self._current_output() + if current is not None: + return current.as_table(columns=columns, all_info=all_info) # Requires a live operator for schema derivation (pre-existing limitation). if self._operator is None: return pa.table({}) - empty_records = self._make_empty_table() - tag_keys = self.keys()[0] - empty_stream = ArrowTableStream(empty_records, tag_columns=tag_keys) + empty_stream = ArrowTableStream( + self._make_empty_table(), + tag_columns=self.keys()[0], + data_context=self.data_context, + ) return empty_stream.as_table(columns=columns, all_info=all_info) def head( @@ -1501,66 +1519,25 @@ def head( columns: ColumnConfig | dict[str, Any] | None = None, all_info: bool = False, ) -> StreamView: - """Return the first rows the node holds or has stored; never runs the operator. + """Return the first rows of the node's current run; never runs the operator. - After a run (or ``execute``) the rows are the node's output in memory. Before - that, a node in ``CacheMode.REPLAY`` shows the first of its stored rows, read - from its whole stored table; in ``LOG`` and ``OFF`` it shows none, as - ``as_table`` does. A loaded node never raises here. See ``StreamBase.head``. + After a run (or ``execute``) the rows are the node's output in memory. Before a + run in this process the node shows no rows and says so, pointing to ``run()`` + and to ``as_source()``, in every cache mode; once stale it says its inputs + changed (ITL-1095). The store is never read. See ``StreamBase.head``. """ check_head_size(n) config = ColumnConfig.handle_config(columns, all_info=all_info) - if self._cached_output_stream is not None: - if not self._loaded and self.is_stale: - # run() would recompute; the kept rows are not the current result. - return self._head_without_rows( - config, ViewStatus.NOT_COMPUTED, note=STALE_NOTE - ) - return head_of(self._cached_output_stream.head(n, columns=config), self) - if self._cache_mode == CacheMode.REPLAY and self._pipeline_database is not None: - return self._head_from_store(n, config) - if self._loaded: - return self._head_without_rows( - config, - ViewStatus.UNAVAILABLE, - note=( - "loaded node whose results are not stored (cache mode OFF)" - if self._cache_mode == CacheMode.OFF - else "loaded node whose stored results are not replayed " - f"(cache mode {self._cache_mode.value.upper()})" - ), - ) - if self._cache_mode == CacheMode.LOG and self._pipeline_database is not None: - note = ( - "not run in this session; rows of earlier runs are stored but not " - "replayed (cache mode LOG)" - ) - else: - note = "not run; cache mode OFF stores nothing" - return self._head_without_rows(config, ViewStatus.NOT_COMPUTED, note=note) - - def _head_from_store(self, n: int, config: ColumnConfig) -> StreamView: - """Return the first ``n`` of this node's stored rows. - - Reads the node's whole stored slice (no bounded read exists yet) through the - same path as ``as_table`` and keeps nothing on the node. A loaded node whose - run's rows cannot be read (its cache table is absent, or none of its rows is the - run's) shows no rows and says why, where ``as_table`` raises. - """ - try: - stream = self._load_cached_stream_from_db() - except _LoadedCacheUnreadableError as exc: + if self._cached_output_stream is None: return self._head_without_rows( - config, ViewStatus.UNAVAILABLE, note=exc.head_note + config, ViewStatus.NOT_COMPUTED, note=JOB_NODE_NOT_RUN_NOTE ) - if stream is None or stream.as_table().num_rows == 0: + if self.is_stale: + # run() would recompute; the kept rows are not the current result. return self._head_without_rows( - config, - ViewStatus.NOTHING_STORED, - # A loaded job cannot run: say what the store holds instead. - note="the store holds no rows for its run" if self._loaded else None, + config, ViewStatus.NOT_COMPUTED, note=STALE_NOTE ) - return stream.head(n, columns=config).for_stream(self, origin="stored") + return head_of(self._cached_output_stream.head(n, columns=config), self) # ------------------------------------------------------------------ # DB retrieval diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 8d7cf8923..a7535d2b1 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -10,6 +10,7 @@ import functools import logging from abc import ABC, abstractmethod +from datetime import datetime from collections.abc import AsyncIterator, Iterator from typing import TYPE_CHECKING, Any @@ -597,10 +598,34 @@ def bound_source(self) -> StreamProtocol | None: @bound_source.setter def bound_source(self, value: StreamProtocol | None) -> None: - """Bind *value* as the concrete source and invalidate both hash caches.""" + """Bind *value* as the concrete source and invalidate both hash caches. + + Binding stamps the slot as modified, so every job node fed by it is stale and + reads empty until it runs again (ITL-1095). + """ self._bound_source = value self._invalidate_content_hash_cache() self._invalidate_pipeline_hash_cache() + self._update_modified_time() + + @property + def last_modified(self) -> datetime | None: + """When the slot's content last changed: its binding or its bound source. + + The later of the time the slot was bound and the bound source's own + ``last_modified`` (a ``CachedSource`` or ``PollingSource`` moves it when it + reads new rows). A bound source that does not know its modification time + leaves the binding time. A source whose rows change without its modified time + moving (a table read lazily, a file rewritten in place) is not seen: build a + new source object and bind it, or clear the dependent nodes. + """ + own = self._modified_time + bound = ( + self._bound_source.last_modified if self._bound_source is not None else None + ) + if own is None or bound is None: + return own if bound is None else bound + return max(own, bound) # ------------------------------------------------------------------ # Identity — delegate to bound source when set diff --git a/src/orcapod/core/nodes/staleness.py b/src/orcapod/core/nodes/staleness.py new file mode 100644 index 000000000..b5320cce4 --- /dev/null +++ b/src/orcapod/core/nodes/staleness.py @@ -0,0 +1,34 @@ +"""Staleness of job nodes across the job graph (ITL-1095). + +A stream is stale when an upstream or its producer was modified after it +(``StreamBase.is_stale``), which looks one level up only. A job node holds the rows of +its current run, so it is stale also when any job node it depends on is stale: a source +rebound two nodes up makes every node below it stale, not only its direct consumer. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + + +def upstream_job_node_stale(upstreams: Iterable[Any]) -> bool: + """Return whether any function or operator job node in ``upstreams`` is stale. + + Each such node answers with its own ``is_stale``, which applies this rule to its + own upstreams in turn, so the whole ancestry is covered. Side-effect job nodes are + not followed yet: their own ``is_stale`` is redefined by ITL-1091. + + Args: + upstreams: The direct upstream streams of a job node. + + Returns: + ``True`` if any upstream function or operator job node is stale. + """ + from orcapod.core.nodes.function_node import FunctionJobNode + from orcapod.core.nodes.operator_node import OperatorJobNode + + return any( + isinstance(upstream, (FunctionJobNode, OperatorJobNode)) and upstream.is_stale + for upstream in upstreams + ) diff --git a/src/orcapod/core/streams/view.py b/src/orcapod/core/streams/view.py index 8417e984f..f440bde36 100644 --- a/src/orcapod/core/streams/view.py +++ b/src/orcapod/core/streams/view.py @@ -46,7 +46,8 @@ class ViewStatus(str, Enum): means the content is empty. PARTIAL: A computation stopped part way; the rows are those computed so far. NOT_COMPUTED: Nothing is computed, or what was computed is out of date. - NOTHING_STORED: A job node whose store holds no rows for it yet. + NOTHING_STORED: A stored view (``node.as_source()``) whose store holds no rows + for its node yet. UNAVAILABLE: No rows can exist here: a pipeline blueprint, an input slot with no source bound, or a source that is not available. """ @@ -63,6 +64,10 @@ class ViewStatus(str, Enum): "not computed: inputs changed since the last computation; run() recomputes it" ) NOTHING_STORED_NOTE = "nothing stored; run the job" +# A job node shows the rows of its current run only; before one it shows none (ITL-1095). +JOB_NODE_NOT_RUN_NOTE = ( + "not run; run() fills it; node.as_source() shows every stored result" +) BLUEPRINT_NOTE = "pipeline blueprint; run it in a PipelineJob" SIDE_EFFECTS_NOTE = ( "not run; run() delivers its side effects; displaying never delivers side effects" diff --git a/tests/test_pipeline/test_job_node_rows_after_run.py b/tests/test_pipeline/test_job_node_rows_after_run.py index fd57128b0..d859e68cd 100644 --- a/tests/test_pipeline/test_job_node_rows_after_run.py +++ b/tests/test_pipeline/test_job_node_rows_after_run.py @@ -5,8 +5,8 @@ the rows of that run's inputs through ``as_table``, ``iter_data``, ``head`` and ``run()``, and so does a consumer built over the node afterwards; the two orchestrators agree row for row. Stored results of other inputs stay stored and reusable. A run that raised defines no -rows: the node then reads as a node with no run in this process (every row stored under -its content hash). +rows: the node then reads as a node with no run in this process, which is empty (ITL-1095); +what was stored stays readable through ``as_source()``. """ from __future__ import annotations @@ -275,13 +275,7 @@ def add_one(x: int) -> int: class TestRaisedRun: - """A run that raised defines no rows: the node reads as a node with no run.""" - - @staticmethod - def _no_run_read(fn: Any, data: list[dict[str, int]], db: Any) -> list[Any]: - fresh = _single(fn.pod, data, db) - fresh.compile() - return _values(fresh.nodes["node"], lambda r: (r["k"], r["y"])) + """A run that raised defines no rows: the node reads as a node with no run (empty).""" @pytest.mark.parametrize( ("kind", "fn", "policy"), @@ -299,12 +293,13 @@ def test_raised_run_reads_as_no_run(self, kind: str, fn: Any, policy: str) -> No with pytest.raises((ValueError, StoreWriteError)): job.run(orchestrator=_orchestrator(kind, policy)) node = job.nodes["node"] - expected = self._no_run_read(fn, ROWS[:4], db) - assert _values(node, lambda r: (r["k"], r["y"])) == expected + assert _values(node, lambda r: (r["k"], r["y"])) == [] reads = _reads(node) - assert reads["head"] == reads["as_table"] - # Nothing stored was dropped: the stored rows of the earlier run are among them. - assert {(3, 31), (4, 41), (5, 51)} <= set(expected) + assert reads["head"] == reads["as_table"] == [] + assert node.is_stale + # Nothing stored was dropped: the stored view holds the earlier run's rows. + stored = {(r["k"], r["y"]) for r in node.as_source().as_table().to_pylist()} + assert {(3, 31), (4, 41), (5, 51)} <= stored class TestDuplicateInputs: From 39aa109406573cce755cb673aa6b7d0094820073 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:33:58 -0800 Subject: [PATCH 06/63] test: read job nodes after a run, not before (ITL-1095) Co-Authored-By: Claude Opus 5.5 --- .../function_pod/test_ephemeral_result.py | 15 ++--- .../test_function_node_arrow_types.py | 7 ++- .../test_function_node_caching.py | 2 + .../test_function_pod_node_stream.py | 33 ++++++++-- .../nodes/test_function_node_iteration.py | 7 ++- tests/test_pipeline/test_node_head.py | 62 ++++++++++++------- tests/test_pipeline/test_node_head_sorting.py | 13 +++- .../test_node_staleness_after_run.py | 6 +- 8 files changed, 98 insertions(+), 47 deletions(-) diff --git a/tests/test_core/function_pod/test_ephemeral_result.py b/tests/test_core/function_pod/test_ephemeral_result.py index b7fb8dbeb..53bec7e79 100644 --- a/tests/test_core/function_pod/test_ephemeral_result.py +++ b/tests/test_core/function_pod/test_ephemeral_result.py @@ -581,8 +581,8 @@ def test_iter_data_serves_result_after_cross_session_recompute(self): iter_data() serves the fresh result without triggering additional computation. Verifies that: - - after clearing the in-memory cache, iter_data() hot-loads the recomputed - result from the pipeline + ephemeral DBs (exercises _load_cached_entries) + - after clearing the node, its next run loads the recomputed result from the + pipeline + ephemeral DBs (exercises _load_cached_entries) - repeated iter_data() calls return the same result without recomputing """ call_count = {"n": 0} @@ -621,11 +621,12 @@ def counting_double(x: int) -> int: node2.execute(stream) assert call_count["n"] == 2 # recomputed once due to cross-session miss - # execute() calls _update_modified_time() at the end, so is_stale is False here. - # Manually clearing forces the iter_data() hot-load branch (_load_cached_entries → - # _fetch_joined_records) without relying on staleness, so this tests the DB join - # path explicitly rather than the in-memory dict fast-path. - node2._cached_output_datas.clear() + # Clearing the node drops its in-memory results; running it again loads the + # recomputed result through the DB join path (_load_cached_entries → + # _fetch_joined_records) rather than the in-memory dict fast-path. A node shows + # rows only after a run (ITL-1095). + node2.clear_cache() + node2.execute(stream) # iter_data() must serve the recomputed result — no additional computation results = list(node2.iter_data()) diff --git a/tests/test_core/function_pod/test_function_node_arrow_types.py b/tests/test_core/function_pod/test_function_node_arrow_types.py index 0a3764a2a..e2253d91f 100644 --- a/tests/test_core/function_pod/test_function_node_arrow_types.py +++ b/tests/test_core/function_pod/test_function_node_arrow_types.py @@ -230,7 +230,7 @@ def test_as_table_mixing_stored_and_new_rows_keeps_tag_type() -> None: def test_store_read_back_keeps_tag_type( name: str, store: str, tmp_path: pathlib.Path ) -> None: - """Reads from the store (``get_all_records``, a fresh job's reads) keep the tag type.""" + """Reads from the store (``get_all_records``, a fresh job's run) keep the tag type.""" database = _new_store(store, tmp_path) table = _source_table(name) _build(database, table).run() @@ -238,11 +238,14 @@ def test_store_read_back_keeps_tag_type( # A new store object sees the committed tables, as a new process would. database = _new_store(store, tmp_path) - fresh = _build(database, table).nodes["double"] + fresh_job = _build(database, table) + fresh = fresh_job.nodes["double"] records = fresh.get_all_records() assert records is not None assert records.schema.field("k").type == _expected(name) + # The node shows rows once its run has read them from the store (ITL-1095). + fresh_job.run() tag, _ = next(iter(fresh.iter_data())) assert tag.as_table().schema.field("k").type == _expected(name) assert fresh.as_table().schema.field("k").type == _expected(name) diff --git a/tests/test_core/function_pod/test_function_node_caching.py b/tests/test_core/function_pod/test_function_node_caching.py index c58ba463b..2e194db9b 100644 --- a/tests/test_core/function_pod/test_function_node_caching.py +++ b/tests/test_core/function_pod/test_function_node_caching.py @@ -231,6 +231,7 @@ def test_phase1_yields_existing_results(self): # Create a second node with the same stream and shared DB stream2 = _make_stream([{"id": 0, "x": 10}, {"id": 1, "x": 20}]) node2, _ = _make_node(stream2, db=db) + node2.run() # a node shows its rows only after a run (ITL-1095) results = list(node2.iter_data()) # All results should come from Phase 1 (DB), not recomputed @@ -319,6 +320,7 @@ def test_all_existing_entry_ids_skipped_in_phase2(self): node2._versioned_pipeline_path ).num_rows + node2.run() results = list(node2.iter_data()) assert len(results) == 2 diff --git a/tests/test_core/function_pod/test_function_pod_node_stream.py b/tests/test_core/function_pod/test_function_pod_node_stream.py index 2ab30e0e7..bda5c9e06 100644 --- a/tests/test_core/function_pod/test_function_pod_node_stream.py +++ b/tests/test_core/function_pod/test_function_pod_node_stream.py @@ -169,7 +169,8 @@ def test_as_table_returns_cached_results_when_data_function_inactive( ): """ Cache filled by node1 (active) is shared with node2 (inactive). - node2.as_table() must return full results from Phase 1 (DB). + node2's run serves the full results from Phase 1 (DB); a node shows its rows + only after a run (ITL-1095). """ n = 3 db = InMemoryArrowDatabase() @@ -181,6 +182,7 @@ def test_as_table_returns_cached_results_when_data_function_inactive( double_pf.set_active(False) node2 = _make_node(double_pf, n=n, db=db) + node2.run() table2 = node2.as_table() assert isinstance(table2, pa.Table) @@ -201,6 +203,7 @@ def test_inactive_filled_db_serves_cached_results(self, double_pf): double_pf.set_active(False) node2 = _make_node(double_pf, n=n, db=db) + node2.run() # a node shows its rows only after a run (ITL-1095) data = list(node2.iter_data()) assert len(data) == n @@ -266,7 +269,9 @@ def test_db_served_results_have_correct_row_count(self, double_pf): n = 5 db = InMemoryArrowDatabase() _fill_node(_make_node(double_pf, n=n, db=db)) - data = list(_make_node(double_pf, n=n, db=db).iter_data()) + node2 = _make_node(double_pf, n=n, db=db) + node2.run() # a node shows its rows only after a run (ITL-1095) + data = list(node2.iter_data()) assert len(data) == n def test_fresh_db_always_computes(self, double_pf): @@ -373,9 +378,11 @@ def counting_double(x: int) -> int: class TestFunctionNodeStaleness: # --- is_stale --- - def test_is_stale_false_immediately_after_creation(self, double_pf): - """A freshly created FunctionNode whose upstream has not changed is not stale.""" + def test_is_stale_immediately_after_creation(self, double_pf): + """A freshly created job node holds no run, so it is stale (ITL-1095).""" node = _make_node(double_pf, n=3) + assert node.is_stale + node.run() assert not node.is_stale def test_is_stale_true_after_upstream_modified(self, double_pf): @@ -395,7 +402,7 @@ def test_is_stale_true_after_upstream_modified(self, double_pf): assert node.is_stale - def test_is_stale_false_after_clear_cache(self, double_pf): + def test_is_stale_after_clear_cache_until_it_runs(self, double_pf): import time db = InMemoryArrowDatabase() @@ -405,13 +412,16 @@ def test_is_stale_false_after_clear_cache(self, double_pf): input_stream=input_stream, pipeline_database=db, ) - list(node.iter_data()) + node.run() time.sleep(0.01) input_stream._update_modified_time() assert node.is_stale + # A cleared node holds no run: it stays stale until it runs (ITL-1095). node.clear_cache() + assert node.is_stale + node.run() assert not node.is_stale # --- clear_cache --- @@ -431,6 +441,9 @@ def test_clear_cache_produces_same_results_on_re_iteration(self, double_pf): table_before = node.as_table() node.clear_cache() + # A cleared node reads empty until it runs again (ITL-1095). + assert node.as_table().num_rows == 0 + node.run() table_after = node.as_table() assert sorted(table_before.column("result").to_pylist()) == sorted( @@ -449,12 +462,17 @@ def test_iter_data_auto_detects_stale_and_repopulates(self, double_pf): input_stream=input_stream, pipeline_database=db, ) + node.run() first = list(node.iter_data()) + assert len(first) == 3 time.sleep(0.01) input_stream._update_modified_time() assert node.is_stale + # A stale node reads empty (ITL-1095); its run reads the stored results again. + assert list(node.iter_data()) == [] + node.run() second = list(node.iter_data()) assert len(second) == len(first) assert [p["result"] for _, p in second] == [p["result"] for _, p in first] @@ -476,6 +494,9 @@ def test_as_table_auto_detects_stale_and_repopulates(self, double_pf): time.sleep(0.01) input_stream._update_modified_time() + # A stale node reads empty (ITL-1095); its run reads the stored results again. + assert node.as_table().num_rows == 0 + node.run() table_after = node.as_table() assert len(table_after) == 3 assert sorted(table_after.column("result").to_pylist()) == sorted( diff --git a/tests/test_core/nodes/test_function_node_iteration.py b/tests/test_core/nodes/test_function_node_iteration.py index fee923c42..e28d54b4d 100644 --- a/tests/test_core/nodes/test_function_node_iteration.py +++ b/tests/test_core/nodes/test_function_node_iteration.py @@ -59,14 +59,17 @@ def test_iter_does_not_call_process_data_internal(self): list(node.iter_data()) mock_proc.assert_not_called() - def test_iter_after_db_populated_hot_loads_without_compute(self): - """iter_data() on a node with DB records hot-loads without _process_data_internal.""" + def test_iter_after_db_populated_yields_nothing_until_a_run_loads(self): + """A node over a populated DB yields nothing before a run (ITL-1095); its run + loads every result from the DB without _process_data_internal.""" db = InMemoryArrowDatabase() node1 = _make_node(n=3, db=db) node1.run() # populate DB node2 = _make_node(n=3, db=db) + assert list(node2.iter_data()) == [] with patch.object(node2, "_process_data_internal") as mock_proc: + node2.run() results = list(node2.iter_data()) mock_proc.assert_not_called() assert len(results) == 3 diff --git a/tests/test_pipeline/test_node_head.py b/tests/test_pipeline/test_node_head.py index 7cbea6a69..e4c821561 100644 --- a/tests/test_pipeline/test_node_head.py +++ b/tests/test_pipeline/test_node_head.py @@ -98,10 +98,12 @@ def test_nodes_before_a_run_show_no_rows_and_compute_nothing() -> None: op_html = job.nodes["sel"]._repr_html_() assert "could not read" not in op_html assert CALLS == [] - assert fn_head.status is ViewStatus.NOTHING_STORED + # A node that has not run says so and names both ways in (ITL-1095). + for head in (fn_head, op_head): + assert head.status is ViewStatus.NOT_COMPUTED + assert "not run" in head.header() + assert "as_source()" in head.header() assert fn_head.table.column_names == ["id", "y"] - assert op_head.status is ViewStatus.NOT_COMPUTED - assert "cache mode OFF" in op_head.header() @pytest.mark.parametrize("store_kind", ["in_memory", "delta"]) @@ -129,26 +131,33 @@ def test_nodes_after_a_run_show_their_first_rows( @pytest.mark.parametrize("store_kind", ["in_memory", "delta"]) -def test_a_new_job_over_a_filled_store_reads_stored_rows_without_computing( +def test_a_new_job_over_a_filled_store_shows_rows_once_its_run_reads_the_store( store_kind: str, tmp_path: Path ) -> None: - """A new job over a filled store shows stored rows without computing.""" + """A new job over a filled store shows no rows until it runs (ITL-1095); its run + reads every result from the store and computes nothing.""" store = _store(store_kind, tmp_path) _ = _job(store).run() CALLS.clear() - node = _job(store).nodes["triple"] + job = _job(store) + node = job.nodes["triple"] + assert node.head(4).status is ViewStatus.NOT_COMPUTED + # The stored view shows the stored rows meanwhile. + assert node.as_source().head(4).status is ViewStatus.ROWS + _ = job.run() head = node.head(4) assert CALLS == [] assert head.status is ViewStatus.ROWS - # The whole stored slice is read, so the header states the exact count and - # makes no claim of a bounded read. - assert head.header() == "FunctionJobNode[triple] — first 4 of 25 stored rows" + assert head.header() == "FunctionJobNode[triple] — first 4 of 25 rows" assert head.table.equals(node.as_table().slice(0, 4)) -def test_a_loaded_job_shows_stored_rows_and_never_raises(tmp_path: Path) -> None: - """A loaded job's nodes show stored rows, or say why there are none.""" +def test_a_loaded_job_shows_no_rows_until_it_runs_and_never_raises( + tmp_path: Path, +) -> None: + """A loaded job's nodes show no rows before a run (ITL-1095) and say where the + stored rows are; ``as_source()`` shows them.""" store = DeltaTableDatabase(base_path=tmp_path / "store") job = _job(store) _ = job.run() @@ -160,10 +169,12 @@ def test_a_loaded_job_shows_stored_rows_and_never_raises(tmp_path: Path) -> None fn_head = loaded.nodes["triple"].head() op_head = loaded.nodes["sel"].head() assert CALLS == [] - assert fn_head.status is ViewStatus.ROWS - assert fn_head.header().endswith("first 10 of 25 stored rows") - assert op_head.status is ViewStatus.UNAVAILABLE - assert "cache mode OFF" in op_head.header() + for head in (fn_head, op_head): + assert head.status is ViewStatus.NOT_COMPUTED + assert "as_source()" in head.header() + stored = loaded.nodes["triple"].as_source().head() + assert stored.status is ViewStatus.ROWS + assert stored.header().endswith("first 10 of 25 stored rows") def test_blueprint_nodes_are_unavailable() -> None: @@ -207,27 +218,27 @@ def _operator_node( def test_operator_node_cache_modes() -> None: - """Operator nodes show stored rows in REPLAY and none in LOG before a run.""" + """Before a run an operator node shows no rows in every cache mode (ITL-1095); a + REPLAY node shows the stored rows once it has run.""" store = InMemoryArrowDatabase() assert _operator_node(CacheMode.REPLAY, store).head().status is ( - ViewStatus.NOTHING_STORED + ViewStatus.NOT_COMPUTED ) seed = _operator_node(CacheMode.LOG, store) _ = seed.execute(_source()) replay = _operator_node(CacheMode.REPLAY, store) + assert replay.head(3).status is ViewStatus.NOT_COMPUTED + _ = replay.run() head = replay.head(3) assert head.status is ViewStatus.ROWS - assert ( - head.header() - == "OperatorJobNode[SelectDataColumns] — first 3 of 25 stored rows" - ) + assert head.header() == "OperatorJobNode[SelectDataColumns] — first 3 of 25 rows" assert head.table.equals(replay.as_table().slice(0, 3)) log = _operator_node(CacheMode.LOG, store) log_head = log.head() assert log_head.status is ViewStatus.NOT_COMPUTED - assert "not replayed" in log_head.header() + assert "as_source()" in log_head.header() @pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) @@ -268,7 +279,7 @@ def test_head_keeps_the_tag_arrow_type_of_the_input( """A node's head keeps an ``int8`` tag ``int8`` and is the prefix of ``as_table``. ``head`` and ``as_table`` share one table build (ITL-928 typing), in memory after a - run and from the store in a new job. + run and in a new job whose run reads its rows from the store. """ table = pa.table( {"id": pa.array(range(12), pa.int8()), "x": pa.array(range(12), pa.int64())}, @@ -293,7 +304,10 @@ def build(store: ArrowDatabaseProtocol) -> PipelineJob: store = _store(store_kind, tmp_path) job = build(store) _ = job.run(orchestrator=ORCHESTRATORS[orchestrator]()) - for node in (job.nodes["triple"], build(store).nodes["triple"]): + # The new job shows rows once its run has read them from the store (ITL-1095). + fresh = build(store) + _ = fresh.run(orchestrator=ORCHESTRATORS[orchestrator]()) + for node in (job.nodes["triple"], fresh.nodes["triple"]): head = node.head(5, all_info=True) assert head.status is ViewStatus.ROWS assert head.table.schema.field("id").type == pa.int8() diff --git a/tests/test_pipeline/test_node_head_sorting.py b/tests/test_pipeline/test_node_head_sorting.py index 42dccf48f..bc4e38793 100644 --- a/tests/test_pipeline/test_node_head_sorting.py +++ b/tests/test_pipeline/test_node_head_sorting.py @@ -121,7 +121,8 @@ def _held_order(node: Any) -> list[int]: def test_a_function_nodes_head_is_the_prefix_of_as_table_in_any_completion_order( run_kind: str, store_kind: str, n: int, config: str, tmp_path: Path ) -> None: - """After a run, and from the store in a new job, head(n, cfg) == as_table(cfg)[:n].""" + """After a run, and in a new job whose run reads every row from the store, + head(n, cfg) == as_table(cfg)[:n].""" store = _store(store_kind, tmp_path) job = _run(run_kind, store) assert _held_order(job.nodes["node"]) != sorted(_held_order(job.nodes["node"])) @@ -131,6 +132,9 @@ def test_a_function_nodes_head_is_the_prefix_of_as_table_in_any_completion_order store, list(range(N_ROWS)), ) + # A node shows its current run only (ITL-1095): the new job's run reads its rows from + # the store, in the store's order, and computes nothing. + fresh.run() for node in (job.nodes["node"], fresh.nodes["node"]): head = node.head(n, **CONFIGS[config]) assert head.status is ViewStatus.ROWS @@ -175,11 +179,14 @@ def node(mode: CacheMode) -> OperatorJobNode: ran = node(CacheMode.LOG) ran.execute(_source(order)) - for each in (ran, node(CacheMode.REPLAY)): + # A REPLAY node shows the stored rows once it has run (ITL-1095). + replayed = node(CacheMode.REPLAY) + replayed.run() + for each in (ran, replayed): head = each.head(3, **CONFIGS[config]) assert head.status is ViewStatus.ROWS assert head.table.equals(each.as_table(**CONFIGS[config]).slice(0, 3)) - sorted_ids = node(CacheMode.REPLAY).head(3, all_info=True).table.column("id") + sorted_ids = replayed.head(3, all_info=True).table.column("id") assert sorted_ids.to_pylist() == [0, 1, 2] diff --git a/tests/test_pipeline/test_node_staleness_after_run.py b/tests/test_pipeline/test_node_staleness_after_run.py index fae00b2dc..9c0e6a23a 100644 --- a/tests/test_pipeline/test_node_staleness_after_run.py +++ b/tests/test_pipeline/test_node_staleness_after_run.py @@ -325,8 +325,8 @@ def test_a_function_node_is_stamped_before_its_output_closes(with_store: bool) - pipeline_database=store, result_database=store, ) - constructed = node.last_modified - assert constructed is not None + # A node that has not run holds nothing and has no modified time (ITL-1095). + assert node.last_modified is None seen_at_close: list[Any] = [] async def run() -> None: @@ -353,4 +353,4 @@ async def read() -> None: asyncio.run(run()) (stamp,) = seen_at_close - assert stamp is not None and stamp > constructed + assert stamp is not None From f1c36f22a3b8160b5dc0d3a1c03650df82efa193 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:35:05 -0800 Subject: [PATCH 07/63] test: read stored results of loaded jobs through as_source() (ITL-1095) Co-Authored-By: Claude Opus 5.5 --- .../test_cached_function_pod_job.py | 5 ++-- .../test_polars_dataframe_values.py | 12 ++++++--- .../function_pod/test_readback_datagram_id.py | 5 +++- .../nodes/test_function_node_empty_data.py | 26 ++++++++++++++----- .../test_provisional_hash_warning.py | 3 ++- tests/test_torch/test_tensor_job_load_back.py | 3 ++- 6 files changed, 39 insertions(+), 15 deletions(-) diff --git a/tests/test_core/function_pod/test_cached_function_pod_job.py b/tests/test_core/function_pod/test_cached_function_pod_job.py index 6ada9c6d4..e14541b98 100644 --- a/tests/test_core/function_pod/test_cached_function_pod_job.py +++ b/tests/test_core/function_pod/test_cached_function_pod_job.py @@ -233,9 +233,10 @@ def test_saved_job_equals_the_uncached_pods_and_loads( doc["bindings"].pop("database") assert cached_saved == plain_saved - # A loaded job reads its stored results; the pod cache is not part of it. + # A loaded job's stored results are read through the stored view (its nodes are + # empty until it runs, ITL-1095); the pod cache is not part of it. loaded = PipelineJob.load(tmp_path / "cached.json", database=store) - assert loaded.nodes["add"].as_table().to_pylist() == [ + assert loaded.nodes["add"].as_source().as_table().to_pylist() == [ {"k": 1, "total": 5}, {"k": 2, "total": 10}, ] diff --git a/tests/test_core/function_pod/test_polars_dataframe_values.py b/tests/test_core/function_pod/test_polars_dataframe_values.py index 170d0f53b..c18a38515 100644 --- a/tests/test_core/function_pod/test_polars_dataframe_values.py +++ b/tests/test_core/function_pod/test_polars_dataframe_values.py @@ -462,19 +462,23 @@ def test_saved_job_loaded_back_reads_the_frames(self, tmp_path: Path) -> None: node = loaded.nodes["make"] _, data_schema = node.output_schema() assert data_schema["df"] is pl.DataFrame - got = {tag["k"]: data["df"] for tag, data in node.iter_data()} + # A loaded job's nodes are empty until it runs (ITL-1095); its stored results + # are read through the stored view. + stored_view = node.as_source() + got = {tag["k"]: data["df"] for tag, data in stored_view.iter_data()} _assert_exact(got["a"], _frame(1)) _assert_exact(got["b"], _frame(3)) stored = dict( zip( - node.as_table().column("k").to_pylist(), - _stored_frames(node.as_table().column("df")), + stored_view.as_table().column("k").to_pylist(), + _stored_frames(stored_view.as_table().column("df")), strict=True, ) ) _assert_exact(stored["a"], _frame(1)) heights = { - tag["k"]: data["height"] for tag, data in loaded.nodes["height"].iter_data() + tag["k"]: data["height"] + for tag, data in loaded.nodes["height"].as_source().iter_data() } assert heights == {"a": 1, "b": 3} diff --git a/tests/test_core/function_pod/test_readback_datagram_id.py b/tests/test_core/function_pod/test_readback_datagram_id.py index 661df3598..9b922b8ae 100644 --- a/tests/test_core/function_pod/test_readback_datagram_id.py +++ b/tests/test_core/function_pod/test_readback_datagram_id.py @@ -89,7 +89,10 @@ def test_a_fresh_job_on_the_store_reads_the_stored_ids() -> None: _ = first.run(orchestrator=SyncPipelineOrchestrator()) stored = stored_ids(first) - fresh = build(store) # never run: every row comes from the store + fresh = build(store) + # A node shows rows only after a run (ITL-1095): this run computes nothing, every + # row comes from the store. + _ = fresh.run(orchestrator=SyncPipelineOrchestrator()) assert read_back_ids(fresh) == stored diff --git a/tests/test_core/nodes/test_function_node_empty_data.py b/tests/test_core/nodes/test_function_node_empty_data.py index 3158dc148..7edf9222d 100644 --- a/tests/test_core/nodes/test_function_node_empty_data.py +++ b/tests/test_core/nodes/test_function_node_empty_data.py @@ -174,6 +174,20 @@ def test_normal_result_wins_over_empty_data_token(self, persistent_node): assert isinstance(data, Data) +def _entries(node: FunctionJobNode) -> list: + """The node's entries for its input rows, as its cache façade returns them. + + These tests drive ``execute_data`` (one row, not a run), so the node shows no rows + of a run (ITL-1095); ``get_cached_results`` gives the entries it holds or has + stored, an ``EmptyData`` placeholder for a result whose ephemeral store is gone. + """ + entry_ids = [ + node.compute_base_entry_id(tag, data) + for tag, data in node._input_stream.iter_data() + ] + return list(node.get_cached_results(entry_ids).values()) + + class TestProcessDataInternalEmptyData: def test_cache_hit_returns_cached_result(self, ephemeral_node): """Downstream cache hit on EmptyData input returns the cached result.""" @@ -203,7 +217,7 @@ def triple_value(result: int) -> int: ) # Step 3: Run downstream with real data to populate its result cache - for tag, data in node.iter_data(): + for tag, data in _entries(node): downstream.execute_data(tag, data) # Step 4: Simulate upstream ephemeral expiry @@ -211,7 +225,7 @@ def triple_value(result: int) -> int: node._cached_output_datas.clear() # Step 5: Upstream now emits EmptyData - upstream_items = list(node.iter_data()) + upstream_items = _entries(node) assert len(upstream_items) == 1 tag_in, data_in = upstream_items[0] assert isinstance(data_in, EmptyData) @@ -252,7 +266,7 @@ def triple_value(result: int) -> int: result_database=InMemoryArrowDatabase(), ) - upstream_items = list(node.iter_data()) + upstream_items = _entries(node) tag_in, data_in = upstream_items[0] assert isinstance(data_in, EmptyData) @@ -296,7 +310,7 @@ def triple_value(result: int) -> int: ) # Populate downstream result cache with real data (sync) - for tag, data in node.iter_data(): + for tag, data in _entries(node): downstream.execute_data(tag, data) # Simulate ephemeral expiry @@ -304,7 +318,7 @@ def triple_value(result: int) -> int: node._cached_output_datas.clear() # Upstream now emits EmptyData - upstream_items = list(node.iter_data()) + upstream_items = _entries(node) assert len(upstream_items) == 1 tag_in, data_in = upstream_items[0] assert isinstance(data_in, EmptyData) @@ -341,7 +355,7 @@ def triple_value(result: int) -> int: result_database=InMemoryArrowDatabase(), ) - upstream_items = list(node.iter_data()) + upstream_items = _entries(node) tag_in, data_in = upstream_items[0] assert isinstance(data_in, EmptyData) diff --git a/tests/test_logical_types/test_provisional_hash_warning.py b/tests/test_logical_types/test_provisional_hash_warning.py index 5a450ee09..c3e102a73 100644 --- a/tests/test_logical_types/test_provisional_hash_warning.py +++ b/tests/test_logical_types/test_provisional_hash_warning.py @@ -232,7 +232,8 @@ def run_and_save() -> None: _quiet(run_and_save) with recorded() as caught: loaded = PipelineJob.load(tmp_path / "job.json") - table = loaded.nodes["make"].as_table() + # A loaded node is empty until it runs (ITL-1095): read the stored view. + table = loaded.nodes["make"].as_source().as_table() assert table.num_rows == 3 assert _warned_names(caught) == [name] diff --git a/tests/test_torch/test_tensor_job_load_back.py b/tests/test_torch/test_tensor_job_load_back.py index 20b819ecd..2d1022c5c 100644 --- a/tests/test_torch/test_tensor_job_load_back.py +++ b/tests/test_torch/test_tensor_job_load_back.py @@ -85,7 +85,8 @@ def test_a_loaded_job_reads_its_stored_tensors( assert loaded.nodes["value"].output_schema()[1]["value"] is torch.Tensor assert loaded.nodes["joined"].load_status == LoadStatus.FULL assert loaded.nodes["joined"].output_schema()[1]["value"] is torch.Tensor - got = _values(loaded.nodes["value"]) + # A loaded node is empty until it runs (ITL-1095): read the stored view. + got = _values(loaded.nodes["value"].as_source()) assert sorted(got) == [1, 2] for row_id, n in ((1, 3), (2, 4)): expected = function(n) From bea357f404010bd2e326e2fcdda120751e2f312c Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:36:45 -0800 Subject: [PATCH 08/63] fix(function_node): count a run input only when it gave a row; docstrings for the new tests (ITL-1093) Addresses the CodeRabbit review of PR #378: of two equal inputs where one fails and the other succeeds, the node now holds one row, as downstream received. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/function_node.py | 13 ++- .../test_job_node_rows_after_run.py | 92 ++++++++++++++++--- 2 files changed, 89 insertions(+), 16 deletions(-) diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index aa9c50dff..252c98c6e 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -1972,6 +1972,9 @@ def execute( # and prevents spurious recomputation of already-processed data. self.get_cached_results(base_entry_ids=base_entry_ids) + # Base entry ids of the inputs that gave a row (a cache hit or a + # successful computation), one per input: the node's rows afterwards. + emitted_entry_ids: list[bytes] = [] for tag, data, base_entry_id in upstream_entries: ctx_obs.on_data_start(node_label, tag, data) @@ -1980,6 +1983,7 @@ def execute( ): tag_out, result = self._cached_output_datas[base_entry_id] ctx_obs.on_data_end(node_label, tag, data, result, cached=True) + emitted_entry_ids.append(base_entry_id) if result is not None: output.append((tag_out, result)) continue @@ -2027,6 +2031,7 @@ def execute( continue ctx_obs.on_data_end(node_label, tag, data, result, cached=False) + emitted_entry_ids.append(base_entry_id) if result is not None: output.append((tag_out, result)) self._commit_or_end_node( @@ -2046,7 +2051,7 @@ def execute( raise # The node's rows are those of this run's inputs, whatever else it caches. - self._set_run_rows(base_entry_ids) + self._set_run_rows(emitted_entry_ids) ctx_obs.on_node_end(node_label, node_hash) # Mark this node as freshly computed so subsequent iter_data() calls # skip the is_stale check and serve results directly from the in-memory cache. @@ -3460,7 +3465,8 @@ async def async_execute( # Local dict: correlation_key → (original_tag, original_input_data) input_store: dict[bytes, tuple[TagProtocol, DataProtocol, bytes]] = {} - # Base entry ids of this run's inputs, in arrival order (ITL-1093). + # Base entry ids of this run's inputs that gave a row (a cache hit or a + # forwarded computed row), one per input (ITL-1093). run_entry_ids: list[bytes] = [] async def route_inputs() -> None: @@ -3468,7 +3474,6 @@ async def route_inputs() -> None: try: async for tag, data in input_channel: base_entry_id = self.compute_base_entry_id(tag, data) - run_entry_ids.append(base_entry_id) # A cached EmptyData is a result that is gone (an # ephemeral result from an earlier run): compute the row # again, as the sync path does (ITL-923). @@ -3485,6 +3490,7 @@ async def route_inputs() -> None: ctx_obs.on_data_end( node_label, tag, data, cached_data, cached=True ) + run_entry_ids.append(base_entry_id) await output.send((cached_tag, cached_data)) else: correlation_key = uuid.uuid4().bytes @@ -3542,6 +3548,7 @@ async def record_and_forward() -> None: # Commits every commit_every_rows rows inside a job (ITL-919); # no await in between, so no other row interleaves. self._row_processed() + run_entry_ids.append(base_entry_id) await output.send((clean_tag, output_data)) # Capture outer self so _NodeLabelObserver methods can diff --git a/tests/test_pipeline/test_job_node_rows_after_run.py b/tests/test_pipeline/test_job_node_rows_after_run.py index fd57128b0..e1a53ee7e 100644 --- a/tests/test_pipeline/test_job_node_rows_after_run.py +++ b/tests/test_pipeline/test_job_node_rows_after_run.py @@ -76,10 +76,12 @@ def store_fails_on_20(x: int) -> int: def _source(data: list[dict[str, int]]) -> DictSource: + """Return a ``DictSource`` of ``data`` with tags ``g`` and ``k`` and source id ``s``.""" return DictSource(data, tag_columns=["g", "k"], source_id="s") def _build(data: list[dict[str, int]], db: Any) -> PipelineJob: + """Return a job over ``db``: ``rowwise``, ``downstream`` of it, and ``grouped``.""" job = PipelineJob(name="itl1093", database=db) with job: rowwise = inc.pod(_source(data), label="rowwise") @@ -89,6 +91,7 @@ def _build(data: list[dict[str, int]], db: Any) -> PipelineJob: def _orchestrator(kind: str, error_policy: str = "continue") -> Any: + """Return the ``sync`` or ``async`` orchestrator with ``error_policy``.""" if kind == "sync": return SyncPipelineOrchestrator(error_policy=error_policy) # pyright: ignore[reportArgumentType] return AsyncPipelineOrchestrator(error_policy=error_policy) # pyright: ignore[reportArgumentType] @@ -121,18 +124,22 @@ def _reads(node: Any) -> dict[str, list[tuple[tuple[str, str], ...]]]: def _values(node: Any, key: Callable[[dict[str, Any]], Any]) -> list[Any]: + """Return ``key`` of every row of ``node.as_table()``, sorted.""" return sorted(key(row) for row in node.as_table().to_pylist()) def _expected_rowwise(data: list[dict[str, int]]) -> list[tuple[int, int]]: + """Return the ``(k, y)`` rows ``rowwise`` gives for ``data``, sorted.""" return sorted((row["k"], row["x"] + 1) for row in data) def _expected_downstream(data: list[dict[str, int]]) -> list[tuple[int, int]]: + """Return the ``(k, z)`` rows ``downstream`` gives for ``data``, sorted.""" return sorted((row["k"], 2 * (row["x"] + 1)) for row in data) def _expected_grouped(data: list[dict[str, int]]) -> list[tuple[int, int]]: + """Return the ``(g, total)`` rows ``grouped`` gives for ``data``, sorted.""" sums: dict[int, int] = {} for row in data: sums[row["g"]] = sums.get(row["g"], 0) + row["x"] @@ -163,6 +170,7 @@ def _check_run(job: PipelineJob, data: list[dict[str, int]]) -> None: def _snapshot(job: PipelineJob) -> dict[str, Any]: + """Return every public read of each function node of ``job``.""" return { label: _reads(job.nodes[label]) for label in ("rowwise", "downstream", "grouped") @@ -174,6 +182,7 @@ class TestFreshJobPerRun: @pytest.mark.parametrize("kind", ORCHESTRATORS) def test_each_run_returns_its_inputs_rows(self, kind: str) -> None: + """Every read after each run gives that run's rows; the full run reuses 3.""" db = InMemoryArrowDatabase() for index, data in enumerate(SEQUENCE): job = _build(data, db) @@ -185,6 +194,7 @@ def test_each_run_returns_its_inputs_rows(self, kind: str) -> None: assert summary.nodes["rowwise"].computed == 3 def test_sync_and_async_agree_row_for_row(self) -> None: + """Both orchestrators give equal rows, system tags included, after every run.""" snapshots: dict[str, list[dict[str, Any]]] = {} for kind in ORCHESTRATORS: db = InMemoryArrowDatabase() @@ -197,6 +207,7 @@ def test_sync_and_async_agree_row_for_row(self) -> None: @pytest.mark.parametrize("kind", ORCHESTRATORS) def test_node_run_returns_its_inputs_rows(self, kind: str) -> None: + """A node's ``run()`` after the job run shows that run's rows.""" db = InMemoryArrowDatabase() for data in SEQUENCE: job = _build(data, db) @@ -212,6 +223,7 @@ class TestSameJobRebound: @pytest.mark.parametrize("kind", ORCHESTRATORS) def test_each_run_returns_its_inputs_rows(self, kind: str) -> None: + """Every read after each run of the rebound job gives that run's rows.""" db = InMemoryArrowDatabase() job = _build(SEQUENCE[0], db) for data in SEQUENCE: @@ -220,6 +232,7 @@ def test_each_run_returns_its_inputs_rows(self, kind: str) -> None: _check_run(job, data) def test_sync_and_async_agree_row_for_row(self) -> None: + """Both orchestrators give equal rows after every run of the rebound job.""" snapshots: dict[str, list[dict[str, Any]]] = {} for kind in ORCHESTRATORS: db = InMemoryArrowDatabase() @@ -233,6 +246,7 @@ def test_sync_and_async_agree_row_for_row(self) -> None: def _single(pod: Any, data: list[dict[str, int]], db: Any) -> PipelineJob: + """Return a job over ``db`` applying ``pod`` to a source of ``data`` as ``node``.""" job = PipelineJob(name="itl1093_single", database=db) with job: pod(_source(data), label="node") @@ -244,6 +258,7 @@ class TestNullAndInactive: @pytest.mark.parametrize("kind", ORCHESTRATORS) def test_null_value_is_a_row(self, kind: str) -> None: + """A row whose output value is null is one of the run's rows.""" db = InMemoryArrowDatabase() _single(null_on_10.pod, ROWS, db).run(orchestrator=_orchestrator(kind)) job = _single(null_on_10.pod, ROWS[:3], db) @@ -256,8 +271,11 @@ def test_null_value_is_a_row(self, kind: str) -> None: @pytest.mark.parametrize("kind", ORCHESTRATORS) def test_inactive_function_gives_no_rows(self, kind: str) -> None: + """A run of an inactive function has no rows, though other results are stored.""" + @function_pod(output_keys="y") def add_one(x: int) -> int: + """Return ``x + 1``.""" return x + 1 db = InMemoryArrowDatabase() @@ -279,6 +297,7 @@ class TestRaisedRun: @staticmethod def _no_run_read(fn: Any, data: list[dict[str, int]], db: Any) -> list[Any]: + """Return the ``(k, y)`` rows a compiled, unrun job over ``db`` reads.""" fresh = _single(fn.pod, data, db) fresh.compile() return _values(fresh.nodes["node"], lambda r: (r["k"], r["y"])) @@ -292,6 +311,7 @@ def _no_run_read(fn: Any, data: list[dict[str, int]], db: Any) -> list[Any]: ], ) def test_raised_run_reads_as_no_run(self, kind: str, fn: Any, policy: str) -> None: + """After a run that raised, the node reads every stored row, as with no run.""" db = InMemoryArrowDatabase() # Store the results of rows 3..5 first, then run rows 0..3, which fails at x=20. _single(fn.pod, ROWS[3:], db).run(orchestrator=_orchestrator(kind)) @@ -315,7 +335,8 @@ class TestDuplicateInputs: """ @staticmethod - def _node() -> Any: + def _node(pod: Any) -> Any: + """Return a node applying ``pod`` to rows (k=0, x=5) twice and (k=2, x=7).""" import pyarrow as pa from orcapod.core.nodes import FunctionJobNode @@ -332,27 +353,36 @@ def _node() -> Any: schema=schema, ) return FunctionJobNode( - inc.pod, # pyright: ignore[reportArgumentType] # the decorator's pod type is not declared as FunctionPodProtocol + pod, ArrowTableStream(table, tag_columns=["k"]), pipeline_database=InMemoryArrowDatabase(), result_database=InMemoryArrowDatabase(), ) - def test_sync(self) -> None: - node = self._node() - view = node.run() - held = sorted((r["k"], r["y"]) for r in node.as_table().to_pylist()) - assert held == [(0, 6), (0, 6), (2, 8)] - assert sorted((r["k"], r["y"]) for r in view.as_table().to_pylist()) == held + @staticmethod + def _flaky_pod() -> Any: + """Return a pod computing ``x + 1`` whose first call for ``x == 5`` raises.""" + calls: list[int] = [] - def test_async(self) -> None: + @function_pod(output_keys="y") + def flaky_inc(x: int) -> int: + """Return ``x + 1``; raise on the first call for ``x == 5``.""" + calls.append(x) + if x == 5 and calls.count(5) == 1: + raise ValueError("transient failure") + return x + 1 + + return flaky_inc.pod + + @staticmethod + def _run_async(node: Any) -> list[tuple[int, int]]: + """Run ``node`` with ``async_execute`` over its upstream; return the rows sent.""" import asyncio from orcapod.channels import Channel - node = self._node() - async def run() -> list[tuple[int, int]]: + """Feed the upstream's rows in and collect what the node sends out.""" inputs = Channel(buffer_size=16) outputs = Channel(buffer_size=16) # The input stream is the node's own upstream (a test of async_execute). @@ -366,6 +396,42 @@ async def run() -> list[tuple[int, int]]: ] return sorted(got) - received = asyncio.run(run()) + return asyncio.run(run()) + + @staticmethod + def _held(node: Any) -> list[tuple[int, int]]: + """Return the node's ``(k, y)`` rows, sorted.""" + return sorted((r["k"], r["y"]) for r in node.as_table().to_pylist()) + + def test_sync(self) -> None: + """Sync: both equal inputs are held, as the run returned them.""" + node = self._node(inc.pod) + view = node.run() + assert self._held(node) == [(0, 6), (0, 6), (2, 8)] + assert sorted((r["k"], r["y"]) for r in view.as_table().to_pylist()) == ( + self._held(node) + ) + + def test_async(self) -> None: + """Async: both equal inputs are held, as they were sent downstream.""" + node = self._node(inc.pod) + received = self._run_async(node) assert received == [(0, 6), (0, 6), (2, 8)] - assert sorted((r["k"], r["y"]) for r in node.as_table().to_pylist()) == received + assert self._held(node) == received + + def test_sync_one_of_two_equal_inputs_fails(self) -> None: + """Sync: an equal input that failed is not held; the one that succeeded is.""" + node = self._node(self._flaky_pod()) + received = sorted( + (tag.as_dict()["k"], data.as_dict()["y"]) + for tag, data in node.execute(node.upstreams[0]) + ) + assert received == [(0, 6), (2, 8)] + assert self._held(node) == received + + def test_async_one_of_two_equal_inputs_fails(self) -> None: + """Async: an equal input that failed is not held; the one that succeeded is.""" + node = self._node(self._flaky_pod()) + received = self._run_async(node) + assert received == [(0, 6), (2, 8)] + assert self._held(node) == received From 55e01eb2cd57781bb84cde2d12d3fd4222e738d7 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:36:58 -0800 Subject: [PATCH 09/63] test: update operator, delta and stored-context reads to the current-run rule (ITL-1095) Co-Authored-By: Claude Opus 5.5 --- .../operators/test_materialize_source_info.py | 2 ++ .../test_core/operators/test_operator_node.py | 3 +++ .../test_operator_node_non_active.py | 9 ++++++-- .../test_delta_arrow_type_roundtrip.py | 21 +++++++++++++------ 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/tests/test_core/operators/test_materialize_source_info.py b/tests/test_core/operators/test_materialize_source_info.py index 59a7c9f8b..057291375 100644 --- a/tests/test_core/operators/test_materialize_source_info.py +++ b/tests/test_core/operators/test_materialize_source_info.py @@ -694,6 +694,7 @@ def test_rerun_after_pre_fix_store(self, monkeypatch: pytest.MonkeyPatch): ) replay = self._node(db, CacheMode.REPLAY) + _ = replay.run() # a node shows rows only after a run (ITL-1095) assert len(list(replay.iter_data())) == 5 def test_v014_store_is_not_read(self, tmp_path: Path): @@ -739,6 +740,7 @@ def test_v014_store_is_not_read(self, tmp_path: Path): tokens = _source_info_tokens(list(_pin_source().iter_data())) assert _stored_source(new) == tokens replay = self._node(db, CacheMode.REPLAY) + _ = replay.run() # a node shows rows only after a run (ITL-1095) assert _source_info_tokens(list(replay.iter_data())) == tokens diff --git a/tests/test_core/operators/test_operator_node.py b/tests/test_core/operators/test_operator_node.py index 1c54d45c6..53cab260f 100644 --- a/tests/test_core/operators/test_operator_node.py +++ b/tests/test_core/operators/test_operator_node.py @@ -375,6 +375,9 @@ def test_replay_from_cache(self, simple_stream, db): node_replay = _make_node( op, (simple_stream,), db=db, cache_mode=CacheMode.REPLAY ) + # A node shows rows only after a run (ITL-1095); REPLAY's run reads the cache. + assert node_replay.as_table().num_rows == 0 + node_replay.run() table = node_replay.as_table() assert table.num_rows == 3 assert "renamed_x" in table.column_names diff --git a/tests/test_core/operators/test_operator_node_non_active.py b/tests/test_core/operators/test_operator_node_non_active.py index 52922a3de..c3215141f 100644 --- a/tests/test_core/operators/test_operator_node_non_active.py +++ b/tests/test_core/operators/test_operator_node_non_active.py @@ -160,8 +160,11 @@ def test_iter_data_replay_mode_no_records_returns_empty( result = list(node.iter_data()) assert result == [] - def test_iter_data_replay_mode_returns_db_contents(self, simple_source, map_op): - """REPLAY + prior LOG run → iter_data returns DB records without run().""" + def test_iter_data_replay_mode_returns_db_contents_after_its_run( + self, simple_source, map_op + ): + """REPLAY + prior LOG run → iter_data returns nothing before the REPLAY node's + run (ITL-1095) and the DB records after it.""" db = InMemoryArrowDatabase() # Populate DB via LOG run node_log = _node(map_op, (simple_source,), db=db, cache_mode=CacheMode.LOG) @@ -171,6 +174,8 @@ def test_iter_data_replay_mode_returns_db_contents(self, simple_source, map_op): node_replay = _node( map_op, (simple_source,), db=db, cache_mode=CacheMode.REPLAY ) + assert list(node_replay.iter_data()) == [] + node_replay.run() result = list(node_replay.iter_data()) assert len(result) == 3 for _, data in result: diff --git a/tests/test_databases/test_delta_arrow_type_roundtrip.py b/tests/test_databases/test_delta_arrow_type_roundtrip.py index f6ba81599..d87183636 100644 --- a/tests/test_databases/test_delta_arrow_type_roundtrip.py +++ b/tests/test_databases/test_delta_arrow_type_roundtrip.py @@ -194,7 +194,9 @@ def _chained_job(root: pathlib.Path, table: pa.Table) -> PipelineJob: def test_a_rerun_on_the_same_store_writes_no_pipeline_rows( name: str, mode: str, tmp_path: pathlib.Path ) -> None: - """The ITL-990 case: source -> pod -> pod, run twice; a fresh job reads 3 rows.""" + """The ITL-990 case: source -> pod -> pod, run twice; a fresh job's run shows 3 rows + and the store holds 3 records per node (a node shows rows only after a run, + ITL-1095).""" from orcapod.pipeline import AsyncPipelineOrchestrator table = _source_table(TAGS[name]) @@ -206,6 +208,7 @@ def test_a_rerun_on_the_same_store_writes_no_pipeline_rows( job.run() fresh = _chained_job(tmp_path, table) + fresh.run() for label in ("double", "increment"): node = fresh.nodes[label] assert node.as_table().num_rows == 3 @@ -217,11 +220,14 @@ def test_a_rerun_on_the_same_store_writes_no_pipeline_rows( @pytest.mark.parametrize("name", list(TAGS)) def test_a_fresh_job_reads_the_tag_type(name: str, tmp_path: pathlib.Path) -> None: - """A new job on the same store reads the tag type through every read of a node.""" + """A new job on the same store, run, reads the tag type through every read of a + node (a node shows rows only after a run, ITL-1095).""" table = _source_table(TAGS[name]) _chained_job(tmp_path, table).run() - fresh = _chained_job(tmp_path, table).nodes["double"] + fresh_job = _chained_job(tmp_path, table) + fresh_job.run() + fresh = fresh_job.nodes["double"] assert fresh.as_table().schema.field("k").type == TAGS[name].type tag, _ = next(iter(fresh.iter_data())) @@ -232,7 +238,8 @@ def test_a_fresh_job_reads_the_tag_type(name: str, tmp_path: pathlib.Path) -> No # Run in a child process by ``test_a_loaded_job_reads_the_tag_type``: load the job -# saved under ``argv[1]`` and print each function node's tag type as JSON. +# saved under ``argv[1]`` and print each function node's stored tag type as JSON (a +# loaded node is empty until it runs, ITL-1095: its stored rows are its stored view). _READ_LOADED_TAG_TYPES = """ import json, pathlib, sys from orcapod import PipelineJob @@ -240,7 +247,7 @@ def test_a_fresh_job_reads_the_tag_type(name: str, tmp_path: pathlib.Path) -> No root = pathlib.Path(sys.argv[1]) job = PipelineJob.load(root / "job.json", database=DeltaTableDatabase(root / "store")) print(json.dumps({ - label: str(job.nodes[label].as_table().schema.field("k").type) + label: str(job.nodes[label].as_source().as_table().schema.field("k").type) for label in ("double", "increment") })) """ @@ -299,7 +306,9 @@ def build(mode: CacheMode) -> PipelineJob: build(CacheMode.LOG).run() - replayed = build(CacheMode.REPLAY).nodes["sel"].as_table() + replay_job = build(CacheMode.REPLAY) + replay_job.run() # a node shows rows only after a run (ITL-1095) + replayed = replay_job.nodes["sel"].as_table() assert replayed.schema.field("k").type == TAGS[name].type From 3301092487d2505b95a378b1295aec41e5073c6d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:40:38 -0800 Subject: [PATCH 10/63] fix(operator_node): the stored view of a loaded operator raises when its run's rows cannot be named (ITL-1095) A loaded operator node read empty before a run, so its stored view (as_source(), get_all_records) is now the way to its run's rows; it raises, as the node's reads did (ITL-1043), when the job file lacks the run's content hash and nothing is stored under the computed one. Also stamps an async REPLAY node after its inputs close, so its replayed rows are not judged stale. Loaded-operator tests read the stored view. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/operator_node.py | 77 ++++++++++------ ...26-11-13-itl-1095-node-current-run-view.md | 32 ++++++- .../loaded_job/test_loaded_operator_cache.py | 92 ++++++++++--------- .../test_pipeline/test_stored_data_context.py | 14 ++- 4 files changed, 141 insertions(+), 74 deletions(-) diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index c4d3a11f8..0c152c34f 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -1203,6 +1203,50 @@ def _make_empty_table(self) -> "pa.Table": self.data_context.type_converter, ) + def _read_run_records(self) -> "pa.Table | None": + """Read the stored rows of this node's run: its content hash, current context. + + A loaded node names its run by the content hash its job file records. A file + saved before Orcapod recorded it leaves the node only the hash the loaded job + computes; when the store holds no rows under that hash, an empty answer would + make stored results look dropped, so this raises instead (ITL-1043). + + Returns: + The run's stored rows (``_read_stored_records``), or ``None`` when the table + does not exist. + + Raises: + RuntimeError: If a loaded node's job file does not record its run's + content hash and the store holds no rows under the hash the loaded + job computes. + """ + stored = self._read_current_context_records() + records = None if stored is None else self._filter_by_content_hash(stored) + if ( + self._loaded + and self._saved_run_content_hash is None + and (records is None or records.num_rows == 0) + ): + found = ( + "is absent from the attached store" + if stored is None + else f"holds {stored.num_rows} stored rows, none of them under the " + f"content hash the loaded job computes ({self.content_hash().to_string()})" + ) + raise _LoadedCacheUnreadableError( + f"OperatorJobNode {self.label!r}: the job file does not record the " + "content hash of this node's run (a file saved before Orcapod recorded " + "it), so the loaded job cannot name the run's rows, and its cache table " + f"({'/'.join(self._stored_table_path)}) {found}. Upgrading such files is " + "left to the migration tool after v0.3 (ITL-1048); until then build the " + "job from code and attach the node with CacheMode.REPLAY to read them.", + head_note=( + "loaded node: the job file does not record its run's content hash " + "(an older file; ITL-1048)" + ), + ) + return records + def _load_cached_stream_from_db(self) -> "ArrowTableStream | None": """Read from DB in CacheMode.REPLAY only, without modifying node state. @@ -1234,31 +1278,7 @@ def _load_cached_stream_from_db(self) -> "ArrowTableStream | None": return None if self._cache_mode != CacheMode.REPLAY: return None - stored = self._read_current_context_records() - records = None if stored is None else self._filter_by_content_hash(stored) - if ( - self._loaded - and self._saved_run_content_hash is None - and (records is None or records.num_rows == 0) - ): - found = ( - "is absent from the attached store" - if stored is None - else f"holds {stored.num_rows} stored rows, none of them under the " - f"content hash the loaded job computes ({self.content_hash().to_string()})" - ) - raise _LoadedCacheUnreadableError( - f"OperatorJobNode {self.label!r}: the job file does not record the " - "content hash of this node's run (a file saved before Orcapod recorded " - "it), so the loaded job cannot name the run's rows, and its cache table " - f"({'/'.join(self._stored_table_path)}) {found}. Upgrading such files is " - "left to the migration tool after v0.3 (ITL-1048); until then build the " - "job from code and attach the node with CacheMode.REPLAY to read them.", - head_note=( - "loaded node: the job file does not record its run's content hash " - "(an older file; ITL-1048)" - ), - ) + records = self._read_run_records() if self._loaded: self._check_expected_rows(0 if records is None else records.num_rows) if records is None: @@ -1568,7 +1588,9 @@ def get_all_records( if self._pipeline_database is None: return None - results = self._read_stored_records() + # A loaded node that cannot name its run's rows raises rather than show the + # stored rows as none (ITL-1043): this is the read behind node.as_source(). + results = self._read_run_records() if self._loaded: # The advisory check of every read of a loaded node (ITL-1043). self._check_expected_rows(0 if results is None else results.num_rows) @@ -1707,6 +1729,9 @@ async def async_execute( for tag, data in self._cached_output_stream.iter_data(): await output.send((tag, data)) _ = await asyncio.gather(*discards) + # The inputs are closed, so every upstream node has stamped itself: + # stamp after them, or the replayed rows would read as stale. + self._update_modified_time() finally: for discard in discards: _ = discard.cancel() diff --git a/superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md b/superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md index de97f4096..6d8904834 100644 --- a/superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md +++ b/superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md @@ -1,6 +1,6 @@ # ITL-1095: a job node shows the current run; load-only run -Status: DIAGNOSIS + PROPOSAL sent to the coordinator (ask), waiting for decisions. No code changed. +Status: decisions received (see section 2). Implementing in four stages, one commit (or more) each, pushed to the draft PR. Base: PR #378 branch head `2e9972e0` (ITL-1093). Branch: `eywalker/itl-1095-design-what-a-job-node-shows-the-current`. Rule (developer, November 13, 2026): a job node itself (`as_table`, `iter_data`, dataframe @@ -57,9 +57,33 @@ involved in `job.run()`. Only `node.run()` reads its upstream node. Experiment (reverted): making every un-run read empty fails 400 of 4,464 job-related tests in 21 files (list in section 5). -## 2. Proposal - -See the ask message (copied in section 6 once answered). +## 2. Decisions (coordinator, November 13, 2026; proposal in 2026-11-13-itl-1095-proposal.md) + +- D1 REPLAY: R1 for the tag (after a run: union of every logged run; only the before-run read + becomes empty). ASKED of the developer. +- D2 (a): load-only uses only the node's own pipeline records; result-store-only hits are not stored. +- D3 (a)+(i): operators run on the rows that reach them (LOG writes nothing); summary and node + status mark every node below a skipped input "inputs incomplete". +- D4: orchestrator passes side-effect node rows through uncalled in load-only; node reads empty. +- D5: NodeRunSummary.not_stored, .inputs_incomplete; RunSummary.load_only; loaded rows in reused; + repr "rowwise: 6 rows: 3 loaded, 3 not stored (skipped)", "summed: 2 rows: 2 loaded (inputs incomplete: rowwise)". +- D6: amend ITL-1081 rule: run() takes no positional arguments; a job node's run() (and + job.run/run_async) takes keyword-only load_only; non-job streams' run() stays argument-less. +- D7 + developer: state-driven run. A node (itself or ancestor) that holds a completed run and is + not stale does not run again: run() returns its held view. Staleness = the existing + modified-time mechanism; a freshly built or loaded job has every node stale. Load-only-filled + is its own status (ViewStatus e.g. LOADED_ONLY, caption "N loaded, M inputs without a stored + result"), never complete: a full run re-runs it in full. A load-only run that finds everything + stays loaded-only by default (ASKED). Cascade = ancestor sub-DAG on the sync orchestrator, one + mechanism shared with job.run. Summary on node.last_run. Commits per row + flush at end. + job.run() behaviour NOT changed (give options). Side-effect re-delivery with + track_completion=False: clear_cache() then run(). Table-backed/polling sources changing without + modified time: known limit, documented. +- D8 loaded jobs as recommended. D9 as recommended. D10 remove NOTHING_STORED. D11 as recommended. + +Order: 1) empty before a run for every node kind + test updates; 2) load_only on job.run/run_async ++ summary; 3) node.run cascade + state rule + status; 4) loaded jobs. Each stage its own commit, +report counts after each. Async tests also on 3.11. Golden unchanged. ## 3. Implementation sketch (after approval) diff --git a/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py b/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py index f8a08d6a0..5a06e27b2 100644 --- a/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py +++ b/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py @@ -4,10 +4,11 @@ second one and stores the join's output (``CacheMode.LOG``). Columns are declared nullable or not. A job file records each column's nullability and the run content hash of every operator node, so the loaded job reproduces each node's pipeline hash, content hash, table -path and output schema, and a loaded operator node attached in ``CacheMode.REPLAY`` reads -the rows its run stored. A file without those fields (``0.1.0``, or a ``0.2.0`` file written -before them) loads as before, lossy: the operator node still looks under its run's path, -and raises rather than show the stored rows as none. +path and output schema, and the stored view (``as_source()``) of a loaded operator node +attached in ``CacheMode.REPLAY`` reads the rows its run stored. A loaded node itself is +empty until it runs (ITL-1095). A file without those fields (``0.1.0``, or a ``0.2.0`` file +written before them) loads as before, lossy: the operator node still looks under its run's +path, and its stored view raises rather than show the stored rows as none. """ from __future__ import annotations @@ -178,17 +179,20 @@ def test_every_node_keeps_its_identity_and_schema_through_save_and_load( def test_loaded_replay_operator_reads_the_rows_its_run_stored( store_kind: str, nullability: str, read: str, tmp_path: pathlib.Path ) -> None: - """A loaded operator node in REPLAY reads exactly the rows read before saving.""" + """A loaded operator node's stored view reads exactly the rows read before saving; + the node itself is empty until it runs (ITL-1095).""" store = _store(store_kind, tmp_path) job = _run(store, nullability) expected = _sorted(job.nodes["joined"].as_table()) node = _load_replaying(_save(job, tmp_path), store).nodes["joined"] + assert node.as_table().num_rows == 0 + stored = node.as_source() if read == "as_table": - assert _sorted(node.as_table()).equals(expected) + assert _sorted(stored.as_table()).equals(expected) elif read == "iter_data": - assert len(list(node.iter_data())) == expected.num_rows + assert len(list(stored.iter_data())) == expected.num_rows else: - assert _sorted(node.head().table).equals(expected) + assert _sorted(stored.head().table).equals(expected) @pytest.mark.parametrize("nullability", list(NULLABILITY)) @@ -214,9 +218,9 @@ def rebuilt(cache_mode: CacheMode) -> PipelineJob: ) return again - assert _sorted(rebuilt(CacheMode.REPLAY).nodes["joined"].as_table()).equals( - expected - ) + replaying = rebuilt(CacheMode.REPLAY) + _ = replaying.run() # a node shows rows only after a run (ITL-1095) + assert _sorted(replaying.nodes["joined"].as_table()).equals(expected) _ = rebuilt(CacheMode.LOG).run() assert set(store.at(JOB_NAME).list_table_paths()) == tables_after_run @@ -238,7 +242,9 @@ def test_without_saved_nullability_the_operator_still_reads_its_runs_table( node = _load_replaying(saved, store).nodes["joined"] assert node.pipeline_hash() == job.nodes["joined"].pipeline_hash() assert node.node_identity_path == job.nodes["joined"].node_identity_path - assert _sorted(node.as_table()).equals(_sorted(job.nodes["joined"].as_table())) + assert _sorted(node.as_source().as_table()).equals( + _sorted(job.nodes["joined"].as_table()) + ) def _drop_both(data: dict[str, Any]) -> None: @@ -259,12 +265,13 @@ def _drop_both(data: dict[str, Any]) -> None: def test_without_the_run_content_hash_the_loaded_operator_raises( store_kind: str, nullability: str, read: str, tmp_path: pathlib.Path ) -> None: - """A node that cannot name its run's rows never shows the stored rows as none.""" + """A node that cannot name its run's rows never shows the stored rows as none: its + stored view raises.""" store = _store(store_kind, tmp_path) job = _run(store, nullability) node = _load_replaying(_save(job, tmp_path, _drop_both), store).nodes["joined"] with pytest.raises(RuntimeError, match=UNKNOWN_KEY) as excinfo: - result = getattr(node, read)() + result = getattr(node.as_source(), read)() if read == "iter_data": _ = list(result) assert "holds 2 stored rows, none of them under the content hash" in str( @@ -279,23 +286,24 @@ def test_without_the_run_content_hash_an_absent_table_raises_too( saved = _save(_run(InMemoryArrowDatabase(), "non_null"), tmp_path, _drop_both) node = _load_replaying(saved, InMemoryArrowDatabase()).nodes["joined"] with pytest.raises(RuntimeError, match=UNKNOWN_KEY) as excinfo: - _ = node.as_table() + _ = node.as_source().as_table() assert "is absent from the attached store" in str(excinfo.value) @pytest.mark.parametrize("store_kind", STORES) -def test_without_the_run_content_hash_head_says_why_it_shows_no_rows( +def test_without_the_run_content_hash_head_never_raises_and_the_stored_view_says_why( store_kind: str, tmp_path: pathlib.Path ) -> None: - """``head()`` never raises on a loaded node: it shows no rows and names the cause.""" + """``head()`` never raises on a loaded node: it has not run (ITL-1095). Its stored + view raises and names the cause.""" store = _store(store_kind, tmp_path) job = _run(store, "nullable_tag") node = _load_replaying(_save(job, tmp_path, _drop_both), store).nodes["joined"] head = node.head() - assert head.status is ViewStatus.UNAVAILABLE + assert head.status is ViewStatus.NOT_COMPUTED assert head.table.num_rows == 0 - assert "does not record its run's content hash" in head.header() - assert "ITL-1048" in head.header() + with pytest.raises(RuntimeError, match=UNKNOWN_KEY): + _ = node.as_source().head() # --------------------------------------------------------------------------- @@ -315,18 +323,19 @@ def _live_replaying(store: ArrowDatabaseProtocol, job: PipelineJob) -> Any: def _assert_reads_as_live_with_no_rows(loaded_node: Any, live_node: Any) -> None: - """The loaded node reads what a running node reads from the same store: no rows.""" + """The loaded node's stored view reads what a running node's reads from the same + store: no rows.""" + loaded_stored = loaded_node.as_source() + live_stored = live_node.as_source() for config in ({}, {"all_info": True}): - table = loaded_node.as_table(**config) - live = live_node.as_table(**config) + table = loaded_stored.as_table(**config) + live = live_stored.as_table(**config) assert table.num_rows == 0 assert table.schema.equals(live.schema), (table.schema, live.schema) - assert list(loaded_node.iter_data()) == [] - head = loaded_node.head() - assert head.status is ViewStatus.NOTHING_STORED + assert list(loaded_stored.iter_data()) == [] + head = loaded_stored.head() assert head.table.num_rows == 0 - assert "the store holds no rows for its run" in head.header() - assert live_node.head().status is ViewStatus.NOTHING_STORED + assert head.status is live_stored.head().status def test_a_store_holding_only_other_runs_rows_gives_an_empty_result( @@ -556,7 +565,8 @@ def source(column: str, values: list[int]) -> ArrowTableSource: def test_a_format_0_1_0_file_finds_its_join_table_and_never_shows_its_rows_as_none() -> ( None ): - """The real 0.1.0 file: the run's path, ``head()`` says why, ``as_table()`` raises.""" + """The real 0.1.0 file: the run's path; the node has not run (ITL-1095) and its + stored view raises.""" store = InMemoryArrowDatabase() run_join = _store_the_fixture_join(store) loaded = PipelineJob.load(str(FORMAT_0_1_0), database=store) @@ -567,13 +577,11 @@ def test_a_format_0_1_0_file_finds_its_join_table_and_never_shows_its_rows_as_no ) assert node.pipeline_hash() == run_join.pipeline_hash() assert node.node_identity_path == run_join.node_identity_path - head = node.head() - assert head.status is ViewStatus.UNAVAILABLE - assert "ITL-1048" in head.header() + assert node.head().status is ViewStatus.NOT_COMPUTED with pytest.raises( RuntimeError, match=r"'Join': the job file does not record .*ITL-1048" ): - _ = node.as_table() + _ = node.as_source().as_table() # --------------------------------------------------------------------------- @@ -596,14 +604,15 @@ def test_get_cached_output_of_a_loaded_node_reads_its_runs_rows( def test_get_cached_output_of_a_loaded_node_never_caches_unreadable_rows_as_none( tmp_path: pathlib.Path, ) -> None: - """``get_cached_output()`` raises as ``as_table()`` does; later reads raise too.""" + """``get_cached_output()`` raises as the stored view does; the node keeps nothing.""" store = InMemoryArrowDatabase() job = _run(store, "nullable_tag") node = _load_replaying(_save(job, tmp_path, _drop_both), store).nodes["joined"] with pytest.raises(RuntimeError, match="the job file does not record"): _ = node.get_cached_output() with pytest.raises(RuntimeError, match="the job file does not record"): - _ = node.as_table() + _ = node.as_source().as_table() + assert node.as_table().num_rows == 0 # --------------------------------------------------------------------------- @@ -640,11 +649,12 @@ def drop_src_a(data: dict[str, Any]) -> None: def _read_all(job: PipelineJob, label: str) -> pa.Table: - """Read a loaded node three ways and return its table.""" - node = job.nodes[label] - _ = list(node.iter_data()) - _ = node.head() - return node.as_table() + """Read a loaded node's stored view three ways and return its table (the node itself + is empty until it runs, ITL-1095).""" + stored = job.nodes[label].as_source() + _ = list(stored.iter_data()) + _ = stored.head() + return stored.as_table() def _loaded(saved: pathlib.Path, store: ArrowDatabaseProtocol) -> PipelineJob: @@ -717,7 +727,7 @@ def test_matching_counts_and_zero_expected_zero_found_do_not_warn( "joined": 0 } node = _load_replaying(empty_saved, InMemoryArrowDatabase()).nodes["joined"] - assert node.as_table().num_rows == 0 + assert node.as_source().as_table().num_rows == 0 def test_deleting_the_section_by_hand_loads_identically_without_a_check( diff --git a/tests/test_pipeline/test_stored_data_context.py b/tests/test_pipeline/test_stored_data_context.py index 49708925a..c91303ac3 100644 --- a/tests/test_pipeline/test_stored_data_context.py +++ b/tests/test_pipeline/test_stored_data_context.py @@ -764,9 +764,15 @@ def test_alone_they_give_no_rows(self, new_store: Callable[[], Store]): def test_function_node_reads_current_rows_only( self, new_store: Callable[[], Store] ): - """Every read method of a function node returns only the rows of ``CURRENT``.""" + """Every read method of a function node returns only the rows of ``CURRENT``. + + A node shows rows only after a run (ITL-1095); the run reads them from the store. + """ store = both_contexts_store(new_store) - node = build_job(store).nodes["dbl"] + job = build_job(store) + _ = job.run() + assert calls == [] + node = job.nodes["dbl"] assert rows_of(node.as_table()) == [2, 4] assert rows_of(node.get_all_records()) == [2, 4] assert sorted(d.as_dict()["double"] for _, d in node.iter_data()) == [2, 4] @@ -980,7 +986,9 @@ def test_function_node_upstream_of_a_refused_sink_is_not_refused( _ = build_job(store, with_sink=True).run() assert delivered == [] - assert rows_of(build_job(store).nodes["dbl"].as_table()) == [2, 4] + # The function node's stored rows are readable (the stored view: a node shows + # rows only after a run, ITL-1095). + assert rows_of(build_job(store).nodes["dbl"].as_source().as_table()) == [2, 4] def test_completions_logged_by_this_version_are_not_delivered_again( self, store: Store, orchestrator: Orchestrator From c8f66c029b89c702f45907edf6a40c26c4fb756f Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:42:47 -0800 Subject: [PATCH 11/63] test(loaded_job): read loaded nodes' stored results through as_source() (ITL-1095) Four tests that need the loaded node's own load path (one row per entry, EmptyData placeholders, stream columns) are strict xfails until loaded.run(load_only=True). Co-Authored-By: Claude Opus 5.5 --- docs/concepts/function-pods.md | 4 +- .../loaded_job/test_loaded_job_readback.py | 102 ++++++++++++------ .../loaded_job/test_loaded_logical_schemas.py | 3 +- 3 files changed, 72 insertions(+), 37 deletions(-) diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 5f414b039..5a0d89d7c 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -687,7 +687,9 @@ job.run() job.save(workdir / "bmi_job.json") loaded = PipelineJob.load(workdir / "bmi_job.json") -print(loaded.nodes["bmi"].as_table().sort_by("subject_id").to_pandas()) +print(loaded.nodes["bmi"].as_table().num_rows) # not run in this process: empty +# 0 +print(loaded.nodes["bmi"].as_source().as_table().sort_by("subject_id").to_pandas()) # subject_id bmi # 0 mouse_01 1756.944444 # 1 mouse_02 2210.000000 diff --git a/tests/test_pipeline/loaded_job/test_loaded_job_readback.py b/tests/test_pipeline/loaded_job/test_loaded_job_readback.py index 962131eb7..50a54b71c 100644 --- a/tests/test_pipeline/loaded_job/test_loaded_job_readback.py +++ b/tests/test_pipeline/loaded_job/test_loaded_job_readback.py @@ -142,7 +142,8 @@ def record() -> None: path = _save_small_job(tmp_path, record) node = PipelineJob.load(path).nodes["stamp"] assert dict(node.output_schema()[1]) == {"t": datetime.datetime} - assert node.as_table().num_rows == 2 + # A loaded node is empty until it runs (ITL-1095); its stored view reads the rows. + assert node.as_source().as_table().num_rows == 2 def test_datetime_output_with_nothing_stored_returns_typed_zero_rows( @@ -189,8 +190,9 @@ def record() -> None: node = PipelineJob.load(_save_small_job(tmp_path, record)).nodes["paths"] assert node.output_schema()[1]["p"] is pathlib.Path - # The stored rows are still readable, with their stored logical type. - table = node.as_table() + # The stored rows are still readable, with their stored logical type (through the + # stored view: a loaded node is empty until it runs, ITL-1095). + table = node.as_source().as_table() assert table.num_rows == 2 assert isinstance(table.schema.field("p").type, pa.ExtensionType) @@ -219,20 +221,25 @@ def record() -> None: Join()(paths, other, label="joined") loaded = PipelineJob.load(_save_small_job(tmp_path, record)) - assert loaded.nodes["paths"].as_table().num_rows == 2 + assert loaded.nodes["paths"].as_source().as_table().num_rows == 2 @pytest.mark.parametrize("label", FUNCTION_NODES) def test_loaded_function_node_as_table_returns_stored_rows( saved: pathlib.Path, label: str ) -> None: - """A loaded node's table equals the live table: schema and values.""" + """A loaded node's stored view equals the live table: schema and values.""" expected = pq.read_table(saved / "expected" / f"{label}.parquet") - table = _load(saved).nodes[label].as_table() + table = _load(saved).nodes[label].as_source().as_table() assert table.schema == expected.schema assert _sorted(table, TAG_OF[label]).equals(_sorted(expected, TAG_OF[label])) +@pytest.mark.xfail( + strict=True, + reason="ITL-1095: a loaded node reads empty until loaded.run(load_only=True), " + "which reads its entries this way (stage 4)", +) @pytest.mark.parametrize("label", FUNCTION_NODES) def test_loaded_function_node_as_table_all_info_has_stream_columns( saved: pathlib.Path, label: str @@ -243,7 +250,7 @@ def test_loaded_function_node_as_table_all_info_has_stream_columns( # source-info columns. (The live FunctionJobNode.as_table() drops those two # groups -- a separate, pre-existing bug the loaded node does not copy.) expected = pq.read_table(saved / "expected" / f"{label}.all_info.parquet") - table = _load(saved).nodes[label].as_table(all_info=True) + table = _load(saved).nodes[label].as_source().as_table(all_info=True) data_column = next(iter(_load(saved).nodes[label].output_schema()[1])) assert "_context_key" in table.column_names assert f"_source_{data_column}" in table.column_names @@ -258,7 +265,7 @@ def test_loaded_function_node_iter_data_yields_stored_rows(saved: pathlib.Path) rows = sorted( ( (tag.as_dict(), data.as_dict()) - for tag, data in _load(saved).nodes["dbl"].iter_data() + for tag, data in _load(saved).nodes["dbl"].as_source().iter_data() ), key=lambda row: row[0]["id"], ) @@ -373,24 +380,32 @@ def test_other_context_records_do_not_satisfy_saved_expectation( node = PipelineJob.load(path, database=other).nodes["dbl"] # The store is the source of truth (ITL-1043): it holds none of the run's # records, so none is read; the job file's count makes it a warning. + # A loaded node is empty until it runs (ITL-1095): its stored view reads the store. + reader = node if read_method == "get_all_records" else node.as_source() with pytest.warns(StoreContentWarning, match="records 2 rows .* holds 0"): - result = getattr(node, read_method)() + result = getattr(reader, read_method)() rows = list(result) if read_method == "iter_data" else result assert rows is None or len(rows) == 0 +@pytest.mark.xfail( + strict=True, + reason="ITL-1095: a loaded node reads empty until loaded.run(load_only=True), " + "which reads its entries this way (stage 4)", +) def test_recomputed_entry_is_read_once_with_its_latest_result( tmp_path: pathlib.Path, ) -> None: """A loaded node reads one row per entry: the greatest recomputation index wins.""" path = _save_small_job(tmp_path, _record_double) node = PipelineJob.load(path).nodes["dbl"] - entry_id = next(iter(node.iter_data()))[0].as_dict()["id"] + # A loaded node is empty until it runs (ITL-1095): read its stored view. + entry_id = next(iter(node.as_source().iter_data()))[0].as_dict()["id"] # Index 2 is stored before index 1, so "last record wins" would pick 111. _append_recomputed_record(node, entry_id, recomputation_index=2, value=222) _append_recomputed_record(node, entry_id, recomputation_index=1, value=111) - reloaded = PipelineJob.load(path).nodes["dbl"] + reloaded = PipelineJob.load(path).nodes["dbl"].as_source() table = reloaded.as_table() assert table.num_rows == 2 rows = {row["id"]: row["dbl"] for row in table.to_pylist()} @@ -418,6 +433,8 @@ def test_live_node_hot_load_reads_the_latest_recomputed_result( fresh = PipelineJob(name="small", database=store) with fresh: _record_double() + # A node shows rows only after a run (ITL-1095): the run loads every stored entry. + _ = fresh.run() values = { t.as_dict()["id"]: d.as_dict()["dbl"] for t, d in fresh.nodes["dbl"].iter_data() } @@ -425,6 +442,11 @@ def test_live_node_hot_load_reads_the_latest_recomputed_result( assert values[entry_id] == 222 +@pytest.mark.xfail( + strict=True, + reason="ITL-1095: a loaded node reads empty until loaded.run(load_only=True), " + "which reads its entries this way (stage 4)", +) def test_loaded_node_with_ephemeral_results_does_not_crash( tmp_path: pathlib.Path, ) -> None: @@ -460,6 +482,11 @@ def make_ephemeral(job: PipelineJob) -> None: ) +@pytest.mark.xfail( + strict=True, + reason="ITL-1095: a loaded node reads empty until loaded.run(load_only=True), " + "which reads its entries this way (stage 4)", +) def test_newer_unavailable_result_does_not_expose_old_persistent_result( tmp_path: pathlib.Path, ) -> None: @@ -503,11 +530,14 @@ def test_loaded_function_node_warns_once_on_a_store_without_its_runs_records( issued once per node. """ node = _load(saved, database=InMemoryArrowDatabase()).nodes["dbl"] + # A loaded node is empty until it runs (ITL-1095): the store is read through its + # stored view, which warns once. + stored = node.as_source() with pytest.warns(StoreContentWarning) as record: - table = node.as_table() - rows = list(node.iter_data()) + table = stored.as_table() + rows = list(stored.iter_data()) records = node.get_all_records() - head = node.head() + head = stored.head() assert table.num_rows == 0 assert table.column_names == ["id", "dbl"] assert rows == [] @@ -532,7 +562,7 @@ def test_store_content_warning_names_a_delta_stores_location( other = tmp_path / "other_store" node = _load(saved, database=DeltaTableDatabase(str(other))).nodes["dbl"] with pytest.warns(StoreContentWarning, match="the store at '") as record: - _ = node.as_table() + _ = node.as_source().as_table() assert str(other) in str(record[0].message) @@ -568,7 +598,7 @@ def double(x: int) -> int: assert node._pipeline_database is not None assert node._pipeline_database.table_exists(node._versioned_pipeline_path) with pytest.warns(StoreContentWarning, match="records 2 rows .* holds 0"): - assert node.as_table().num_rows == 0 + assert node.as_source().as_table().num_rows == 0 def test_saved_file_records_the_rows_each_node_had_in_the_store( @@ -598,7 +628,7 @@ def test_saving_a_loaded_job_keeps_the_expected_rows( assert run["expected_rows"] == original node = PipelineJob.load(str(resaved), database=InMemoryArrowDatabase()).nodes["dbl"] with pytest.warns(StoreContentWarning, match="records 2 rows"): - assert node.as_table().num_rows == 0 + assert node.as_source().as_table().num_rows == 0 def test_loaded_function_node_with_nothing_stored_returns_zero_rows( @@ -634,8 +664,9 @@ def drop_section(data: dict[str, Any]) -> None: with warnings.catch_warnings(): warnings.simplefilter("error", StoreContentWarning) job = PipelineJob.load(path, database=InMemoryArrowDatabase()) - assert job.nodes["dbl"].as_table().num_rows == 0 - assert PipelineJob.load(path).nodes["dbl"].as_table().num_rows == 2 + assert job.nodes["dbl"].as_source().as_table().num_rows == 0 + stored = PipelineJob.load(path).nodes["dbl"].as_source() + assert stored.as_table().num_rows == 2 def test_a_file_with_the_old_stored_records_flag_loads_without_a_check( @@ -689,7 +720,7 @@ def drop_store(data: dict[str, Any]) -> None: job = PipelineJob.load(_rewrite_job_file(saved, tmp_path, drop_store)) job.bind(database=DeltaTableDatabase(str(saved / "store"))) - assert job.nodes["dbl"].as_table().num_rows == 2 + assert job.nodes["dbl"].as_source().as_table().num_rows == 2 def test_loaded_job_still_cannot_run(saved: pathlib.Path) -> None: @@ -799,21 +830,19 @@ async def run() -> list[Any]: # --------------------------------------------------------------------------- -def test_loaded_operator_node_raises_instead_of_returning_empty( +def test_loaded_operator_node_reads_empty_and_says_it_has_not_run( saved: pathlib.Path, ) -> None: - """A loaded operator node, with nothing stored, raises and names its inputs.""" + """A loaded operator node, like every node, is empty until it runs (ITL-1095): no + read raises, and its head points to ``run()`` and ``as_source()``.""" + from orcapod.core.streams import ViewStatus + node = _load(saved).nodes["joined"] - expected = ( - "OperatorJobNode 'joined' has no stored results available for this read. " - "Operator outputs require a stored cache and replay configuration. Read the stored results of " - "its input nodes ('dbl', 'asrc') instead; an input source whose data was not " - "saved must be bound first with job.bind(sources=...)." - ) - for read in (node.as_table, lambda: list(node.iter_data())): - with pytest.raises(RuntimeError) as excinfo: - _ = read() - assert str(excinfo.value) == expected + assert node.as_table().num_rows == 0 + assert list(node.iter_data()) == [] + head = node.head() + assert head.status is ViewStatus.NOT_COMPUTED + assert "as_source()" in head.header() # --------------------------------------------------------------------------- @@ -867,7 +896,7 @@ def move_csv(data: dict[str, Any]) -> None: "source node 'csrc' itself, bind a source: job.bind(sources={'csrc': })." ) # The stored results downstream of the missing source are still readable. - assert job.nodes["inc"].as_table().num_rows == 2 + assert job.nodes["inc"].as_source().as_table().num_rows == 2 def test_unreconstructable_store_is_a_warning_with_the_remedy( @@ -976,5 +1005,8 @@ def timestamp(x: int) -> int: assert result is not None expected = result["value"] node.clear_cache() - assert node.as_table().column("value").to_pylist() == [expected] - assert [result["value"] for _, result in node.iter_data()] == [expected] + # A cleared node is empty until it runs (ITL-1095); the readback of the entry, the + # lookup a run makes, picks the newest recomputation from whichever store holds it. + assert node.as_table().num_rows == 0 + loaded = node.get_cached_results([node.compute_base_entry_id(tag, data)]) + assert [result["value"] for _, result in loaded.values()] == [expected] diff --git a/tests/test_pipeline/loaded_job/test_loaded_logical_schemas.py b/tests/test_pipeline/loaded_job/test_loaded_logical_schemas.py index b4b90b38d..09f204881 100644 --- a/tests/test_pipeline/loaded_job/test_loaded_logical_schemas.py +++ b/tests/test_pipeline/loaded_job/test_loaded_logical_schemas.py @@ -127,7 +127,8 @@ def test_registered_logical_output_keeps_join_available( loaded = op.PipelineJob.load(path) assert loaded.nodes["value"].output_schema()[1]["value"] is expected_type - assert loaded.nodes["value"].as_table().num_rows == 1 + # A loaded node is empty until it runs (ITL-1095): read its stored view. + assert loaded.nodes["value"].as_source().as_table().num_rows == 1 assert loaded.nodes["joined"].load_status == LoadStatus.FULL assert loaded.nodes["joined"].output_schema()[1]["value"] is expected_type From da66b836fd8fa025345131614c5e3841c3f65795 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:45:13 -0800 Subject: [PATCH 12/63] docs: a job node shows its current run, as_source() what is stored (ITL-1095) Concept pages, PipelineJob.load docstring, CLAUDE.md and .zed/rules rule (replaces the ITL-1093 sentence), changelog fragment and design issue CR1. Staleness across the job graph moves into a RunHoldingNode mixin (no import cycle). Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 18 +++-- CLAUDE.md | 18 +++-- changelog.d/ITL-1095.md | 9 +++ design_issues/ITL-1095-CR1.md | 24 +++++++ docs/concepts/function-pods.md | 69 ++++++++++--------- docs/concepts/sources.md | 4 +- docs/concepts/streams.md | 9 +-- src/orcapod/core/nodes/function_node.py | 23 ++----- src/orcapod/core/nodes/operator_node.py | 15 +--- src/orcapod/core/nodes/staleness.py | 33 ++++++--- src/orcapod/pipeline/job.py | 13 ++-- .../loaded_job/test_loaded_job_readback.py | 3 +- 12 files changed, 144 insertions(+), 94 deletions(-) create mode 100644 changelog.d/ITL-1095.md create mode 100644 design_issues/ITL-1095-CR1.md diff --git a/.zed/rules b/.zed/rules index 2d923a544..6653ea945 100644 --- a/.zed/rules +++ b/.zed/rules @@ -773,13 +773,19 @@ data. The rules: by the user is never changed. SourceStreamBuilder.build has no bare-hash default: it needs a source_id or a source_kind. - ArrowTableStream requires at least one data column; raises ValueError otherwise. -- After a run, a FunctionJobNode returns exactly the rows of that run's inputs, sync or - async (ITL-1093; a run that raised defines no rows). Read without a completed run in this - process (a fresh or loaded job), it returns every record stored under its content hash for - the current data context, not only its current inputs. A run skips inputs whose entry is - already stored. +- A job node shows its current run; node.as_source() shows everything stored for it + (developer, November 13, 2026, ITL-1095). After a run, a FunctionJobNode returns exactly + the rows of that run's inputs, sync or async (ITL-1093). Before a run in this process (a + fresh or loaded job), after a run that raised, after clear_cache(), and while stale, every + function and operator job node reads empty (zero typed rows; head() is not_computed and + points to run() and as_source()), in every cache mode: no read touches the store. A + never-run or cleared node has no modified time, so it is stale; a job node is stale also + when any function or operator job node it depends on is (core/nodes/staleness.py), and a + SourceJobNode is modified when it is bound or its bound source is. A run skips inputs + whose entry is already stored. A loaded operator's stored view raises when its run's rows + cannot be named (ITL-1043). - Empty data → ArrowTableSource raises ValueError("Table is empty"). -- DerivedSource over a job node before run() → presents zero rows with the origin's columns +- DerivedSource over a job node with nothing stored → presents zero rows with the origin's columns (no computed records; no error), in the plain form and with tags_as_data. - Join requires non-overlapping data columns; raises InputValidationError on collision. - MergeJoin requires colliding data columns to have identical Python types; merges into sorted diff --git a/CLAUDE.md b/CLAUDE.md index d075910b4..66a284382 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -814,13 +814,19 @@ data. The rules: by the user is never changed. `SourceStreamBuilder.build` has no bare-hash default: it needs a `source_id` or a `source_kind`. - `ArrowTableStream` requires at least one data column; raises `ValueError` otherwise. -- After a run, a `FunctionJobNode` returns exactly the rows of that run's inputs, sync or - async (ITL-1093; a run that raised defines no rows). Read without a completed run in this - process (a fresh or loaded job), it returns every record stored under its content hash for - the current data context, not only its current inputs. A run skips inputs whose entry is - already stored. +- A job node shows its current run; `node.as_source()` shows everything stored for it + (developer, November 13, 2026, ITL-1095). After a run, a `FunctionJobNode` returns exactly + the rows of that run's inputs, sync or async (ITL-1093). Before a run in this process (a + fresh or loaded job), after a run that raised, after `clear_cache()`, and while stale, every + function and operator job node reads empty (zero typed rows; `head()` is `not_computed` and + points to `run()` and `as_source()`), in every cache mode: no read touches the store. A + never-run or cleared node has no modified time, so it is stale; a job node is stale also + when any function or operator job node it depends on is (`core/nodes/staleness.py`), and a + `SourceJobNode` is modified when it is bound or its bound source is. A run skips inputs + whose entry is already stored. A loaded operator's stored view raises when its run's rows + cannot be named (ITL-1043). - Empty data → `ArrowTableSource` raises `ValueError("Table is empty")`. -- `DerivedSource` over a job node before `run()` → presents zero rows with the origin's columns +- `DerivedSource` over a job node with nothing stored → presents zero rows with the origin's columns (no computed records; no error), in the plain form and with `tags_as_data`. - Join requires non-overlapping data columns; raises `InputValidationError` on collision. - MergeJoin requires colliding data columns to have identical Python types; merges into sorted diff --git a/changelog.d/ITL-1095.md b/changelog.d/ITL-1095.md new file mode 100644 index 000000000..0a5c89eeb --- /dev/null +++ b/changelog.d/ITL-1095.md @@ -0,0 +1,9 @@ +### Breaking changes and migration + +- A job node shows the current run; `node.as_source()` shows everything stored for it (developer, November 13, 2026; ITL-1095). Before a run in this process, a function or operator job node is empty: `as_table()`, `iter_data()`, the dataframe methods, `flow()`, `head()` and the notebook preview return zero rows with the node's columns, and `head()` (status `not_computed`) says the node has not run and points to `run()` and to `node.as_source()`. Before, a function node built over a store that already held results, or loaded with `PipelineJob.load`, returned every row stored under its key, and an operator node in `CacheMode.REPLAY` returned every logged run. The same holds after `clear_cache()`, after a run that raised, and while the node is stale. Migration: read stored results with `node.as_source()` (or `get_all_records()`), or run the job first; a run over a store that holds every result computes nothing. +- A function or operator job node that has not run in this process is stale (`is_stale` is `True`; it has no modified time), and a job node is stale also when any function or operator job node it depends on is. Binding a source to a job's source slot (`job.bind(sources=...)`) marks the slot modified, so every node fed by it is stale and empty until it runs again. +- A loaded operator node no longer raises from `as_table()` or `iter_data()`: it is empty, like every node that has not run. Its stored view (`as_source()`, `get_all_records()`) raises instead when the job file does not record its run's content hash and the store holds nothing under the hash the loaded job computes (ITL-1043). + +### Fixed + +- On the asynchronous orchestrator, an operator node in `CacheMode.REPLAY` stamps itself after its inputs close, so the rows it replayed are not judged out of date by the time the run ends. diff --git a/design_issues/ITL-1095-CR1.md b/design_issues/ITL-1095-CR1.md new file mode 100644 index 000000000..b2e853b2c --- /dev/null +++ b/design_issues/ITL-1095-CR1.md @@ -0,0 +1,24 @@ +## `src/orcapod/core/nodes/function_node.py` + +### CR1 — A job node read without a run showed every stored row, not the current run +**Status:** in progress +**Severity:** high +**Issue:** ITL-1095 + +A job node is read through the same methods whether or not it has run in this process. After +a run it held the rows of that run's inputs (ITL-1093), but read without one it fell back to the +store: a `FunctionJobNode` hot-loaded every record stored under its content hash (the results of +every earlier run over the same `source_id`, including inputs no longer in the source), a loaded +job's function nodes did the same, and an `OperatorJobNode` in `CacheMode.REPLAY` returned every +logged run. Which view a read gave depended on whether a run had happened in the process, and +nothing said so. A fresh or cleared node also reported `is_stale == False`, and staleness looked +one level up only, so a source rebound two nodes up left the nodes below it "current". + +**Fix:** (developer, November 13, 2026) a job node shows its current run and `node.as_source()` +shows everything stored. Function and operator job nodes read empty (typed zero rows, `head()` +`not_computed` pointing to `run()` and `as_source()`) until a run in this process, after +`clear_cache()`, after a run that raised and while stale; no read touches the store. A never-run +or cleared node has no modified time (stale); `core/nodes/staleness.py` makes a job node stale +when any function or operator job node above it is; a `SourceJobNode` is modified when bound or +when its bound source is. Planned in the same issue: `run(load_only=True)` and the `node.run()` +cascade. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 5a0d89d7c..d32956901 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -333,12 +333,19 @@ database. Each recorded step becomes a node, available afterwards as `job.nodes[ listed under its `source_id`). A node is a stream, so `as_table()`, `iter_data()` and `output_schema()` all work on it. -After `job.run()`, a function node returns exactly the rows of that run's inputs, on either -orchestrator. The database may hold more under the same node: results of an earlier run -over other rows of the source stay stored and are reused when those rows come back, but -they are not rows of this run. A run that raised defines no rows. A node read with no -finished run in this process (a job built but not run, or a loaded job) returns every row -stored under its key. +A job node shows the current run; `node.as_source()` shows everything stored for it +(developer, November 13, 2026). After `job.run()`, a function node returns exactly the rows +of that run's inputs, on either orchestrator. The database may hold more under the same +node: results of an earlier run over other rows of the source stay stored and are reused +when those rows come back, but they are not rows of this run. Before a run in this process +(a job built over a store that already holds results, or a loaded job) every node is empty: +`as_table()`, `iter_data()`, `head()` and the preview show no rows, and the preview says the +node has not run and points to `run()` and to `as_source()`. A run fills it, also when every +result is read from the store and nothing is computed. A run that raised defines no rows, +and so does `clear_cache()`; a node whose input changed since its run (an upstream node ran +again, or a source was bound again) is stale and empty until it runs again. The stored view, +`node.as_source()`, reads every record stored under the node's key, whether or not the node +has run. `InMemoryArrowDatabase` lasts only as long as the Python process. Use `DeltaTableDatabase("some/directory")` from `orcapod.databases` to keep results on disk @@ -686,7 +693,7 @@ with job: job.run() job.save(workdir / "bmi_job.json") -loaded = PipelineJob.load(workdir / "bmi_job.json") +loaded = PipelineJob.load(workdir / "bmi_job.json") # its nodes are empty until it runs print(loaded.nodes["bmi"].as_table().num_rows) # not run in this process: empty # 0 print(loaded.nodes["bmi"].as_source().as_table().sort_by("subject_id").to_pandas()) @@ -696,29 +703,29 @@ print(loaded.nodes["bmi"].as_source().as_table().sort_by("subject_id").to_pandas # 2 mouse_03 2297.520661 ``` -A loaded job is for reading stored results. It cannot be run again: function pods are not -saved with the job, so `loaded.run()` raises. To compute again, build the job from your code -as before. - -The store is the source of truth: a loaded job shows what the store holds for it. The job -file names each run's rows by the content hash the run stored them under, so a loaded node -reads exactly what the same node built from your code would read from that store, and no -rows under that key is an empty result, not an error. - -- A function node's `as_table()`, `iter_data()` and `get_all_records()` return the rows - stored under the saved run's key, with the types they were stored with: those of every - run of that node over the same sources, not only the saved run's inputs. Rows come in the order the store - returns them; sort them if order matters. +A loaded job is for reading stored results, through each node's stored view, +`node.as_source()`: its nodes have not run in this process, so they are empty. It cannot be +run again: function pods are not saved with the job, so `loaded.run()` raises. To compute +again, build the job from your code as before. + +The store is the source of truth: a loaded job's stored views show what the store holds for +it. The job file names each run's rows by the content hash the run stored them under, so a +loaded node's stored view reads exactly what the same node built from your code would read +from that store, and no rows under that key is an empty result, not an error. + +- A function node's `as_source()` and `get_all_records()` return the rows stored under the + saved run's key, with the types they were stored with: those of every run of that node + over the same sources, not only the saved run's inputs. Rows come in the order the store + returns them; sort them if order matters. The node itself (`as_table()`, `iter_data()`, + `head()`) is empty: it has not run in this process. - Operators (a join, a filter) store outputs only when caching is enabled, under `odb_v1`. The saved file does not record an operator's cache mode (follow-up ITL-977), so attach a loaded operator node in `CacheMode.REPLAY` to read the rows its run stored: `loaded.nodes["joined"].attach_databases(pipeline_database=store.at(""), - cache_mode=CacheMode.REPLAY)`. A loaded operator node reads exactly as the same node built - from your code and attached in `CacheMode.REPLAY`; when the store holds no rows under the - node's key (no table, or none of its rows), the answer is an empty result, as for that - node. A loaded operator not attached in `REPLAY` has no readable cache and raises from - `as_table()` and `iter_data()`; read its input nodes instead (an in-memory input source - must be bound first, see below), or build the job from your code. + cache_mode=CacheMode.REPLAY)`, then read its stored view, `as_source()`, which reads + exactly as the same node built from your code would; when the store holds no rows under + the node's key (no table, or none of its rows), the answer is an empty result, as for that + node. A loaded operator node itself is empty, like every node that has not run. - A loaded node has the pipeline hash, content hash, table path and output schema its run had (a source node whose data was not saved gets its run's content hash once its source is bound). The file records which columns may be null (`T | None`), because the Arrow type text @@ -727,8 +734,8 @@ rows under that key is an empty result, not an error. file's format version: a file without them (format `0.1.0`, or a `0.2.0` file written before they existed) still loads, but lossy: its schemas read every column as non-null, and a loaded operator node cannot name its run's rows. When the store holds none under the - content hash the loaded job computes, `as_table()` and `iter_data()` raise, and `head()` - shows no rows and says why, rather than presenting stored rows as none. An empty record is + content hash the loaded job computes, its stored view (`as_source()`, `get_all_records()`) + raises and says why, rather than presenting stored rows as none. An empty record is not a missing one: it says that no column may be null. Saving such a loaded job again does not record what its file did not. Upgrading such files is left to a migration tool after v0.3. @@ -741,9 +748,9 @@ rows under that key is an empty result, not an error. `PipelineJob.load(path, database=...)` if the store has moved. - The job file may also hold an optional, advisory section, `run.expected_rows`: for each node, how many rows the store held under its run's key when the job was saved (written - only when that is certain: after a run that finished). Reading a loaded node always - returns what the store holds; when the store holds another number of rows, the read - issues one `StoreContentWarning` per node, naming the node, both numbers and the store: + only when that is certain: after a run that finished). Reading a loaded node's stored + view always returns what the store holds; when the store holds another number of rows, + the read issues one `StoreContentWarning` per node, naming the node, both numbers and the store: the store may not be the one the run wrote, or it may have changed since. Deleting the section from the file changes nothing but this check. diff --git a/docs/concepts/sources.md b/docs/concepts/sources.md index 83d4a8319..b24d1fc6f 100644 --- a/docs/concepts/sources.md +++ b/docs/concepts/sources.md @@ -167,8 +167,8 @@ names them. To see more rows, use `as_df()` with a `limit=` or `where=` source. **Restrictions are not part of identity.** `where`, `limit` and `asof` choose rows of the same table, so two sources on one table have the same `source_id` and `content_hash` whatever their restrictions, and a row's result is computed once and shared. One -consequence: with a shared store, reading a job's node back (`get_all_records()`, a loaded -job, and `as_table()` after an async run) returns every row that any job with the same +consequence: with a shared store, reading a job node's stored records back +(`get_all_records()`, `node.as_source()`) returns every row that any job with the same pipeline has read from that table, not only the rows of the latest restriction. The rows sent downstream during a run are only the rows that run read. diff --git a/docs/concepts/streams.md b/docs/concepts/streams.md index 7d0250146..838cc31fe 100644 --- a/docs/concepts/streams.md +++ b/docs/concepts/streams.md @@ -205,7 +205,7 @@ What `run()` does on each kind of object: | A `PollingSource` | polls, as `as_table()` does | | A `CachedSource` | reads its cache and writes the wrapped source's new rows to it | | A function pod's or operator's output stream | computes what is not computed yet | -| A job node | runs the node as a job run would (`as_table()` of a job node only reads), then returns the view | +| A job node | runs the node as a job run would (`as_table()` of a job node only reads), then returns the view of that run's rows | | A side-effect stream | delivers its side effects for every row, on every call, and returns the rows it passed on; `head()` still shows none | | A pipeline blueprint node, an unbound input slot | raises, as `as_table()` does | @@ -215,7 +215,7 @@ What each kind of object shows: |---|---| | A source | its first rows (a bounded read for a source that reads its table lazily) | | An operator's or function pod's output stream | the rows already computed, or none with status `not_computed` | -| A job node | the rows it holds after `job.run()`, else the first of its stored rows, else none (`nothing_stored`) | +| A job node | the rows of its current run (after `job.run()` or `node.run()`); before a run in this process, or once stale, none (`not_computed`), pointing to `run()` and to `node.as_source()`, the view of every stored result | | A pipeline blueprint node, an unbound input slot | none (`unavailable`) | | A side-effect stream | none: showing it never delivers its side effects | @@ -240,7 +240,7 @@ print(unordered.head(2, all_info=True).as_table().column("subject_id").to_pylist # ['mouse_01', 'mouse_02'] ``` -A stream that has all of its rows (a job node's stored rows, a computed stream, an in-memory +A stream that has all of its rows (a job node after its run, a computed stream, an in-memory source) then gives exactly the first `n` rows of `as_table` with the same configuration, whatever order the rows were computed in. A stream that has only some of its rows sorts those: a source that reads a bounded number of rows (such as a `SpiralDBTableSource`) shows @@ -260,7 +260,8 @@ rows, can be missing from it even when `columns` or `all_info` asks for them. A stream or node records that its results are current when it has read its input to the end. So after `job.run()` on either orchestrator, sync or async, every node shows the rows of that run: `head()`, `as_table()` and `iter_data()` serve them from memory, without reading the -store (a function node whose run left no rows in memory reads its stored rows). Likewise, once a chain of streams such as `g.pod(f.pod(source))` has been read to the +store. Before its run in this process a job node holds nothing and shows no rows, whatever +its store holds; `node.as_source()` reads the store. Likewise, once a chain of streams such as `g.pod(f.pod(source))` has been read to the end, reading it again computes nothing. A change upstream (`clear_cache()`) makes every stream after it out of date, and its next read computes it again. Two reads of one stream open at the same time, or a read that picks up one left partway, share diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 7b28f3b4d..cb8a5ced9 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -42,7 +42,7 @@ StreamView, ) from orcapod.core.tracker import DEFAULT_TRACKER_MANAGER -from orcapod.core.nodes.staleness import upstream_job_node_stale +from orcapod.core.nodes.staleness import RunHoldingNode from orcapod.core.datagrams.tag_data import Data, EmptyData, Tag from orcapod.errors import ( EphemeralResultMissingError, @@ -1035,7 +1035,7 @@ class _JoinedRecords(NamedTuple): empty_taginfo_rows: dict[bytes, dict] -class FunctionJobNode(FunctionNodeBase): +class FunctionJobNode(RunHoldingNode, FunctionNodeBase): """DB-backed execution node for function pod invocations. Created by ``PipelineJob`` at run time; never recorded inside a plain @@ -1529,17 +1529,6 @@ def clear_cache(self) -> None: # No content established: the node, and every node downstream, is stale. self._set_modified_time(None) - @property - def is_stale(self) -> bool: - """Whether the rows this node holds are out of date, or it holds none. - - True before the node's first run in this process and after ``clear_cache()`` - (no modified time), when an input or the function pod was modified after the node's - run (``StreamBase.is_stale``), or when any job node it depends on is stale - (ITL-1095). A stale node reads empty until it runs again. - """ - return super().is_stale or upstream_job_node_stale(self.upstreams) - def _held_pairs(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: """Yield the (tag, data) rows this node holds; every read of them goes through here. @@ -1626,7 +1615,7 @@ def _cannot_compute_error(self) -> RuntimeError: return RuntimeError( f"FunctionJobNode {self.label!r} was loaded by PipelineJob.load and has no " "function pod, so it cannot compute; read its stored results with " - "as_table() or iter_data()." + "as_source()." ) def _stored_record_count(self) -> int: @@ -3241,8 +3230,8 @@ def run(self) -> StreamView: Computes as a job run computes this node, unlike ``as_table()``, which only reads. A node whose upstream is unavailable (``CACHE_ONLY``) or whose function - pod is a placeholder (``READ_ONLY``) computes nothing: the view holds its stored - rows. + pod is a placeholder (``READ_ONLY``) computes nothing and holds no rows; its + stored rows are read with ``as_source()``. Returns: A ``StreamView`` of every row ``as_table()`` then returns. It says "rows", @@ -3263,7 +3252,7 @@ def run(self) -> StreamView: if self._load_status in (LoadStatus.CACHE_ONLY, LoadStatus.READ_ONLY): # CACHE_ONLY: upstream unavailable; computation requires a live input stream. # READ_ONLY: function pod is a proxy placeholder — cannot compute. - # The view serves the existing DB results, as iter_data() does. + # Neither runs, so the view holds no rows (ITL-1095). return self._view_of_rows(self.as_table(), "held") if self.is_stale: # Discard any stale in-memory entries before a fresh computation run diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 0c152c34f..7ca6367d4 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -40,7 +40,7 @@ head_of, ) from orcapod.core.tracker import DEFAULT_TRACKER_MANAGER -from orcapod.core.nodes.staleness import upstream_job_node_stale +from orcapod.core.nodes.staleness import RunHoldingNode from orcapod.errors import ( PipelineJobRequiredError, StoreReadError, @@ -621,7 +621,7 @@ def as_node(self) -> OperatorNode: # --------------------------------------------------------------------------- -class OperatorJobNode(OperatorNodeBase): +class OperatorJobNode(RunHoldingNode, OperatorNodeBase): """DB-backed execution node for operator pod invocations. Stream node representing an operator invocation with optional DB persistence. @@ -1480,17 +1480,6 @@ def run(self) -> StreamView: origin = "computed" if kept is None else head_of(kept.head(1), self).origin return self._view_of_rows(table, origin) - @property - def is_stale(self) -> bool: - """Whether the rows this node holds are out of date, or it holds none. - - True before the node's first run in this process and after ``clear_cache()`` - (no modified time), when an input or the operator was modified after the node's - run (``StreamBase.is_stale``), or when any job node it depends on is stale - (ITL-1095). A stale node reads empty until it runs again. - """ - return super().is_stale or upstream_job_node_stale(self.upstreams) - def _current_output(self) -> StreamProtocol | None: """The output of the node's current run, or ``None`` when it holds none. diff --git a/src/orcapod/core/nodes/staleness.py b/src/orcapod/core/nodes/staleness.py index b5320cce4..06efbe5b8 100644 --- a/src/orcapod/core/nodes/staleness.py +++ b/src/orcapod/core/nodes/staleness.py @@ -12,23 +12,40 @@ from typing import Any +class RunHoldingNode: + """Mixin of the job nodes that hold the rows of their current run. + + ``FunctionJobNode`` and ``OperatorJobNode`` mix it in ahead of their stream base, so + their ``is_stale`` covers their whole ancestry. Side-effect job nodes do not yet: + their own ``is_stale`` is redefined by ITL-1091. + """ + + @property + def is_stale(self) -> bool: + """Whether the rows this node holds are out of date, or it holds none. + + True before the node's first run in this process and after ``clear_cache()`` + (no modified time), when an input or its producer was modified after the + node's run (``StreamBase.is_stale``), or when any job node it depends on is + stale (ITL-1095). A stale node reads empty until it runs again. + """ + own: bool = super().is_stale # pyright: ignore[reportAttributeAccessIssue] # the stream base after this mixin in the MRO defines it + return own or upstream_job_node_stale(self.upstreams) # pyright: ignore[reportAttributeAccessIssue] # every node class this is mixed into has upstreams + + def upstream_job_node_stale(upstreams: Iterable[Any]) -> bool: - """Return whether any function or operator job node in ``upstreams`` is stale. + """Return whether any job node in ``upstreams`` that holds run rows is stale. Each such node answers with its own ``is_stale``, which applies this rule to its - own upstreams in turn, so the whole ancestry is covered. Side-effect job nodes are - not followed yet: their own ``is_stale`` is redefined by ITL-1091. + own upstreams in turn, so the whole ancestry is covered. Args: upstreams: The direct upstream streams of a job node. Returns: - ``True`` if any upstream function or operator job node is stale. + ``True`` if any upstream ``RunHoldingNode`` is stale. """ - from orcapod.core.nodes.function_node import FunctionJobNode - from orcapod.core.nodes.operator_node import OperatorJobNode - return any( - isinstance(upstream, (FunctionJobNode, OperatorJobNode)) and upstream.is_stale + isinstance(upstream, RunHoldingNode) and upstream.is_stale for upstream in upstreams ) diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index c38146edc..e71cf1607 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -1396,10 +1396,12 @@ def load( """Deserialize a ``PipelineJob`` from a JSON file to read its stored results. A loaded job is for reading the results its saved run stored; it cannot be - run again, because function pods are not saved with the job. Read a - function node with ``job.nodes[label].as_table()``, ``iter_data()`` or - ``get_all_records()``. Operators can store outputs when caching is enabled; - reading a loaded operator without a readable cache raises ``RuntimeError``. + run again, because function pods are not saved with the job. Its nodes have + not run in this process, so they are empty (ITL-1095): read a node's stored + results through its stored view, ``job.nodes[label].as_source()``, or with + ``get_all_records()``. Operators store outputs only when caching is enabled; + the stored view of a loaded operator whose run's rows cannot be named raises + ``RuntimeError``. The embedded ``Pipeline`` blueprint is always restored. Sources whose data lives in files (CSV, Delta) are rebuilt from the saved config. In-memory @@ -1415,7 +1417,8 @@ def load( over any database configuration in the file. Returns: - A read-only ``PipelineJob`` whose nodes serve their stored results. + A read-only ``PipelineJob`` whose nodes' stored views serve their stored + results. Raises: ValueError: If the file's format version is unsupported, or if it was diff --git a/tests/test_pipeline/loaded_job/test_loaded_job_readback.py b/tests/test_pipeline/loaded_job/test_loaded_job_readback.py index 50a54b71c..b14dff36e 100644 --- a/tests/test_pipeline/loaded_job/test_loaded_job_readback.py +++ b/tests/test_pipeline/loaded_job/test_loaded_job_readback.py @@ -801,8 +801,7 @@ def test_loaded_function_node_cannot_execute(saved: pathlib.Path) -> None: _ = job.nodes["dbl"].execute(job.nodes["dsrc"]) assert str(excinfo.value) == ( "FunctionJobNode 'dbl' was loaded by PipelineJob.load and has no function " - "pod, so it cannot compute; read its stored results with as_table() or " - "iter_data()." + "pod, so it cannot compute; read its stored results with as_source()." ) From 7411a897fd8645a89aab80a52fdfca5059d2c21a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:46:50 -0800 Subject: [PATCH 13/63] test(pipeline): failing tests: job.run(load_only=True) (ITL-1095) Co-Authored-By: Claude Opus 5.5 --- tests/test_pipeline/test_load_only_run.py | 232 ++++++++++++++++++++++ 1 file changed, 232 insertions(+) create mode 100644 tests/test_pipeline/test_load_only_run.py diff --git a/tests/test_pipeline/test_load_only_run.py b/tests/test_pipeline/test_load_only_run.py new file mode 100644 index 000000000..f3c1ff5c9 --- /dev/null +++ b/tests/test_pipeline/test_load_only_run.py @@ -0,0 +1,232 @@ +"""``job.run(load_only=True)``: fill the nodes from what is stored, compute nothing (ITL-1095). + +Developer, November 13, 2026: "it might help for run to have an argument ... load_only=True +where it would fill the pipeline only based on what can be loaded ... and does not trigger +computation of any missing entries." A load-only run reads the sources, runs operators on +the rows that reach them, and fills each function node with the results already stored for +this run's inputs. No user function is called, no side effect is delivered, nothing is +written to the store. Inputs without a stored result are skipped and counted. +""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.side_effects import SideEffectPod, tap_pod +from orcapod.sources import DictSource +from orcapod.types import CacheMode + +CALLS: list[str] = [] +DELIVERED: list[int] = [] + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + """Return ``x + 1``.""" + CALLS.append("inc") + return x + 1 + + +@function_pod(output_keys="total") +def total(y: list[int]) -> int: + """Return the sum of a group's values.""" + CALLS.append("total") + return sum(y) + + +def _deliver(y: int, ctx: Any) -> None: + """A side effect: record the delivered value.""" + DELIVERED.append(y) + + +ROWS = [{"g": i % 2, "k": i, "x": 10 * i} for i in range(6)] +ORCHESTRATORS = {"sync": SyncPipelineOrchestrator, "async": AsyncPipelineOrchestrator} + + +@pytest.fixture(autouse=True) +def _reset() -> None: + CALLS.clear() + DELIVERED.clear() + + +def _build( + data: list[dict[str, int]], + db: Any, + cache_mode: CacheMode | None = None, + with_tap: bool = False, +) -> PipelineJob: + """``rowwise`` (function) -> ``grouped`` (GroupBy) -> ``summed`` (function). + + With ``with_tap``, a side-effect node ``tap`` sits between ``rowwise`` and the group. + """ + src = DictSource(data, tag_columns=["g", "k"], source_id="s") + job = PipelineJob(name="j", database=db) + with job: + rows: Any = inc.pod(src, label="rowwise") + if with_tap: + tap = tap_pod(_deliver) + assert isinstance(tap, SideEffectPod) + rows = tap.process(rows, label="tap") + grouped = rows.group_by("g", label="grouped") + total.pod(grouped, label="summed") + if cache_mode is not None: + job.nodes["grouped"].attach_databases( + pipeline_database=db.at("j"), cache_mode=cache_mode + ) + return job + + +def _snapshot(db: InMemoryArrowDatabase) -> dict[tuple[str, ...], int]: + """Every table of the store with its row count.""" + db.flush() + counts: dict[tuple[str, ...], int] = {} + for path in db.list_table_paths(): + records = db.get_all_records(path) + counts[path] = 0 if records is None else records.num_rows + return counts + + +def _run(job: PipelineJob, orchestrator: str, **kwargs: Any) -> Any: + return job.run(orchestrator=ORCHESTRATORS[orchestrator](), **kwargs) + + +def _totals(job: PipelineJob) -> list[tuple[int, int]]: + return sorted( + (r["g"], r["total"]) for r in job.nodes["summed"].as_table().to_pylist() + ) + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_a_load_only_run_over_a_full_store_equals_a_run_and_computes_nothing( + orchestrator: str, +) -> None: + db = InMemoryArrowDatabase() + _run(_build(ROWS, db), orchestrator) + expected = _build(ROWS, InMemoryArrowDatabase()) + _run(expected, orchestrator) + CALLS.clear() + before = _snapshot(db) + + job = _build(ROWS, db) + summary = _run(job, orchestrator, load_only=True) + + assert CALLS == [] + assert _snapshot(db) == before + assert job.nodes["rowwise"].as_table().num_rows == 6 + assert _totals(job) == _totals(expected) + assert summary.load_only + rowwise = summary.nodes["rowwise"] + assert (rowwise.computed, rowwise.reused, rowwise.not_stored) == (0, 6, 0) + assert rowwise.inputs_incomplete == () + assert summary.nodes["summed"].inputs_incomplete == () + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_inputs_without_a_stored_result_are_skipped_and_counted( + orchestrator: str, +) -> None: + """The store holds a 3-row run; the source now has 6 rows.""" + db = InMemoryArrowDatabase() + _run(_build(ROWS[:3], db), orchestrator) + CALLS.clear() + before = _snapshot(db) + + job = _build(ROWS, db) + summary = _run(job, orchestrator, load_only=True) + + assert CALLS == [] + assert _snapshot(db) == before + assert sorted(r["k"] for r in job.nodes["rowwise"].as_table().to_pylist()) == [ + 0, + 1, + 2, + ] + rowwise = summary.nodes["rowwise"] + assert (rowwise.computed, rowwise.reused, rowwise.not_stored) == (0, 3, 3) + # The groups below hold only the loaded rows: those of the earlier 3-row run, whose + # totals are stored. The summary says their inputs are incomplete. + assert _totals(job) == [(0, 22), (1, 11)] + summed = summary.nodes["summed"] + assert (summed.reused, summed.not_stored) == (2, 0) + assert summed.inputs_incomplete == ("rowwise",) + text = repr(summary) + assert "rowwise: 6 rows: 3 loaded, 3 not stored (skipped)" in text + assert "summed: 2 rows: 2 loaded (inputs incomplete: rowwise)" in text + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_a_load_only_run_over_an_empty_store_fills_nothing_and_writes_nothing( + orchestrator: str, +) -> None: + db = InMemoryArrowDatabase() + job = _build(ROWS, db) + summary = _run(job, orchestrator, load_only=True) + assert CALLS == [] + assert _snapshot(db) == {} + for label in ("rowwise", "grouped", "summed"): + assert job.nodes[label].as_table().num_rows == 0, label + assert summary.nodes["rowwise"].not_stored == 6 + assert summary.nodes["summed"].reused == 0 + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_a_load_only_run_delivers_no_side_effect_and_passes_the_rows_on( + orchestrator: str, +) -> None: + db = InMemoryArrowDatabase() + _run(_build(ROWS, db, with_tap=True), orchestrator) + assert len(DELIVERED) == 6 + DELIVERED.clear() + CALLS.clear() + before = _snapshot(db) + + job = _build(ROWS, db, with_tap=True) + _run(job, orchestrator, load_only=True) + + assert DELIVERED == [] + assert CALLS == [] + assert _snapshot(db) == before + assert _totals(job) == [(0, 63), (1, 93)] + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_a_log_operator_writes_nothing_in_a_load_only_run(orchestrator: str) -> None: + db = InMemoryArrowDatabase() + _run(_build(ROWS[:3], db, cache_mode=CacheMode.LOG), orchestrator) + before = _snapshot(db) + + job = _build(ROWS[:3], db, cache_mode=CacheMode.LOG) + _run(job, orchestrator, load_only=True) + + assert _snapshot(db) == before + assert job.nodes["grouped"].as_table().num_rows == 2 + + +def test_run_async_takes_load_only() -> None: + db = InMemoryArrowDatabase() + _run(_build(ROWS, db), "sync") + CALLS.clear() + job = _build(ROWS, db) + summary = asyncio.run(job.run_async(load_only=True)) + assert CALLS == [] + assert summary.load_only + assert job.nodes["rowwise"].as_table().num_rows == 6 + + +def test_load_only_is_keyword_only() -> None: + job = _build(ROWS, InMemoryArrowDatabase()) + with pytest.raises(TypeError): + job.run(None, None, True, 100, True) # pyright: ignore[reportCallIssue] # load_only is keyword-only + + +def test_a_full_run_summary_is_not_load_only() -> None: + summary = _build(ROWS, InMemoryArrowDatabase()).run() + assert not summary.load_only + assert summary.nodes["rowwise"].not_stored == 0 + assert summary.nodes["rowwise"].inputs_incomplete == () From 6541425ce28ef62244520b4cb60e2d22b6719aac Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:52:16 -0800 Subject: [PATCH 14/63] feat(pipeline): job.run(load_only=True) fills the nodes from what is stored (ITL-1095) job.run, job.run_async and both orchestrators take keyword-only load_only. Sources are read, operators run on the rows that reach them (a LOG cache writes nothing), each function node takes the results stored for this run's inputs (its own records under its content hash) and skips and counts the others, and a side-effect node passes its rows on uncalled. No function call, no delivery, no write. RunSummary.load_only, NodeRunSummary.not_stored and .inputs_incomplete; the text and HTML displays say what was loaded and skipped. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/function_node.py | 174 +++++++++++++++++- src/orcapod/core/nodes/operator_node.py | 15 +- src/orcapod/pipeline/async_orchestrator.py | 48 +++-- src/orcapod/pipeline/job.py | 44 +++-- src/orcapod/pipeline/run_summary.py | 118 +++++++++++- src/orcapod/pipeline/sync_orchestrator.py | 30 ++- src/orcapod/protocols/node_protocols.py | 4 + .../nodes/test_function_node_empty_data.py | 3 +- .../test_job_node_current_run_view.py | 2 +- 9 files changed, 391 insertions(+), 47 deletions(-) diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index cb8a5ced9..e331de106 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -1062,6 +1062,10 @@ class FunctionJobNode(RunHoldingNode, FunctionNodeBase): # its number of inputs; None when no run completed (ITL-1093). A class default so # nodes rebuilt from a saved descriptor start without a run. _run_entry_counts: Counter[bytes] | None = None + # Whether the last completed run was load-only, and how many of its inputs had no + # stored result (ITL-1095). Class defaults for nodes rebuilt from a descriptor. + _run_load_only: bool = False + _run_not_stored: int = 0 def __init__( self, @@ -1526,6 +1530,8 @@ def clear_cache(self) -> None: self._cached_output_table = None self._cached_content_hash_column = None self._run_entry_counts = None + self._run_load_only = False + self._run_not_stored = 0 # No content established: the node, and every node downstream, is stale. self._set_modified_time(None) @@ -1552,12 +1558,162 @@ def _held_pairs(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: if data is not None: yield tag, data - def _set_run_rows(self, entry_ids: list[bytes]) -> None: - """Make the inputs of a completed run, by base entry id, the rows this node holds.""" + def _set_run_rows( + self, entry_ids: list[bytes], *, load_only: bool = False, not_stored: int = 0 + ) -> None: + """Make the inputs of a completed run, by base entry id, the rows this node holds. + + Args: + entry_ids: The base entry ids of the run's inputs whose rows the node holds. + load_only: Whether the run was load-only (ITL-1095). + not_stored: How many inputs of a load-only run had no stored result. + """ self._run_entry_counts = Counter(entry_ids) + self._run_load_only = load_only + self._run_not_stored = not_stored self._cached_output_table = None self._cached_content_hash_column = None + @property + def last_run_not_stored(self) -> int: + """Inputs the node's last completed run skipped because nothing was stored. + + Non-zero only after a load-only run (``run(load_only=True)``); 0 before a run. + """ + return self._run_not_stored + + def _stored_entry( + self, entry_id: bytes + ) -> tuple[TagProtocol, DataProtocol | None] | None: + """The result held for ``entry_id`` after a lookup, or ``None`` if none is stored. + + A placeholder for a result kept in an ephemeral store that is gone is not a + stored result. + """ + entry = self._cached_output_datas.get(entry_id) + if entry is None or isinstance(entry[1], EmptyData): + return None + return entry + + def _execute_load_only( + self, + input_stream: StreamProtocol, + observer: ExecutionObserverProtocol | None, + ) -> list[tuple[TagProtocol, DataProtocol]]: + """Fill the node with the stored results of ``input_stream``'s rows; compute nothing. + + The load-only form of ``execute`` (ITL-1095): each input's entry is looked up + among the node's own stored entries (its pipeline records under its content + hash, the lookup a run makes before computing). An input with a stored result + is a loaded row (``on_data_start`` and ``on_data_end(cached=True)``); one + without is skipped and counted (``last_run_not_stored``), with no observer + call. No function is called and nothing is written. + + Returns: + The loaded (tag, output data) rows, excluding ``None`` outputs. + + Raises: + RuntimeError: If no store is attached (``UNAVAILABLE``). + InputValidationError: If the input stream is of another data context. + """ + from orcapod.pipeline.observer import NoOpObserver + from orcapod.pipeline.serialization import LoadStatus + + if self.load_status == LoadStatus.UNAVAILABLE: + raise self._no_store_error() + require_same_data_context( + input_stream.data_context_key, self._stored_data_context_key + ) + node_label = self.label + node_hash = self.content_hash().to_string() + obs = observer if observer is not None else NoOpObserver() + ctx_obs = obs.contextualize(*self.node_identity_path) + tag_schema = input_stream.output_schema(columns={"system_tags": True})[0] + ctx_obs.on_node_start(node_label, node_hash, tag_schema=tag_schema) + output: list[tuple[TagProtocol, DataProtocol]] = [] + loaded_ids: list[bytes] = [] + not_stored = 0 + try: + inputs = [ + (tag, data, self.compute_base_entry_id(tag, data)) + for tag, data in input_stream.iter_data() + ] + _ = self.get_cached_results(base_entry_ids=[eid for _, _, eid in inputs]) + for tag, data, entry_id in inputs: + entry = self._stored_entry(entry_id) + if entry is None: + not_stored += 1 + continue + ctx_obs.on_data_start(node_label, tag, data) + ctx_obs.on_data_end(node_label, tag, data, entry[1], cached=True) + loaded_ids.append(entry_id) + if entry[1] is not None: + output.append((entry[0], entry[1])) + except BaseException: + # A run that raised defines no rows. + self.clear_cache() + raise + self._set_run_rows(loaded_ids, load_only=True, not_stored=not_stored) + ctx_obs.on_node_end(node_label, node_hash) + self._update_modified_time() + return output + + async def _async_execute_load_only( + self, + input_channel: ReadableChannel[tuple[TagProtocol, DataProtocol]], + output: WritableChannel[tuple[TagProtocol, DataProtocol]], + observer: ExecutionObserverProtocol | None, + ) -> None: + """The async form of ``_execute_load_only``: same rows, counts and observer calls. + + The node reads its whole input before it looks the entries up, then sends the + loaded rows; it stamps itself before its output closes (ITL-1042), and closes + the output whatever happens. + """ + from orcapod.pipeline.observer import NoOpObserver + from orcapod.pipeline.serialization import LoadStatus + + try: + if self.load_status == LoadStatus.UNAVAILABLE: + raise self._no_store_error() + require_same_data_context( + self._input_stream.data_context_key, self._stored_data_context_key + ) + node_label = self.label + node_hash = self.content_hash().to_string() + obs = observer if observer is not None else NoOpObserver() + ctx_obs = obs.contextualize(*self.node_identity_path) + tag_schema = self._input_stream.output_schema( + columns={"system_tags": True} + )[0] + ctx_obs.on_node_start(node_label, node_hash, tag_schema=tag_schema) + inputs = [ + (tag, data, self.compute_base_entry_id(tag, data)) + async for tag, data in input_channel + ] + _ = self.get_cached_results(base_entry_ids=[eid for _, _, eid in inputs]) + loaded_ids: list[bytes] = [] + not_stored = 0 + for tag, data, entry_id in inputs: + entry = self._stored_entry(entry_id) + if entry is None: + not_stored += 1 + continue + ctx_obs.on_data_start(node_label, tag, data) + ctx_obs.on_data_end(node_label, tag, data, entry[1], cached=True) + loaded_ids.append(entry_id) + if entry[1] is not None: + await output.send((entry[0], entry[1])) + self._set_run_rows(loaded_ids, load_only=True, not_stored=not_stored) + ctx_obs.on_node_end(node_label, node_hash) + self._update_modified_time() + except BaseException: + # A run that raised or was cancelled defines no rows. + self.clear_cache() + raise + finally: + await output.close() + def set_ephemeral_store(self, store: "ArrowDatabaseProtocol | None") -> None: """Assign or remove the ephemeral result store. @@ -1912,6 +2068,7 @@ def execute( observer: ExecutionObserverProtocol | None = None, error_policy: Literal["continue", "fail_fast"] = "continue", run_id: str | None = None, + load_only: bool = False, ) -> list[tuple[TagProtocol, DataProtocol]]: """Execute all data from a stream: compute, persist, and cache. @@ -1931,6 +2088,11 @@ def execute( ``"fail_fast"`` re-raises on the first failure. run_id: Optional pipeline run identifier threaded through to ``InvocationContext.pipeline_run_id``. + load_only: Fill the node with the results already stored for the + stream's rows and compute nothing (ITL-1095): no function call, no + write; inputs without a stored result are skipped and counted + (``last_run_not_stored``). A loaded node (no function pod) can run + this way. Returns: Materialized list of (tag, output_data) pairs, excluding @@ -1945,6 +2107,8 @@ def execute( """ from orcapod.pipeline.observer import NoOpObserver + if load_only: + return self._execute_load_only(input_stream, observer) if self._function_pod is None: raise self._cannot_compute_error() @@ -3274,6 +3438,7 @@ async def async_execute( *, observer: ExecutionObserverProtocol | None = None, run_id: str | None = None, + load_only: bool = False, ) -> None: """Streaming async execution for FunctionJobNode. @@ -3296,6 +3461,8 @@ async def async_execute( observer: Optional execution observer for hooks. run_id: Optional pipeline run identifier threaded through to ``InvocationContext.pipeline_run_id``. + load_only: As for ``execute``: fill the node with the stored results of + its input rows and compute nothing (ITL-1095). Raises: RuntimeError: If the node was loaded by ``PipelineJob.load``: without a @@ -3304,6 +3471,9 @@ async def async_execute( """ from orcapod.pipeline.serialization import LoadStatus + if load_only: + await self._async_execute_load_only(input_channel, output, observer) + return try: self._begin_run() except StoreError: diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 7ca6367d4..16c3321d6 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -1322,12 +1322,17 @@ def execute( self, *input_streams: StreamProtocol, observer: ExecutionObserverProtocol | None = None, + load_only: bool = False, ) -> list[tuple[TagProtocol, DataProtocol]]: """Execute input streams: compute, persist, and cache. Args: *input_streams: Input streams to execute. observer: Optional execution observer for hooks. + load_only: A load-only run (ITL-1095): the operator still runs on the rows + that reach it (it is not a stored computation), but nothing is + written: a ``LOG`` node does not store its output. ``REPLAY`` replays + as in any run. Returns: Materialized list of (tag, data) pairs. @@ -1368,9 +1373,10 @@ def execute( self._update_modified_time() - # Persist to DB only in LOG mode + # Persist to DB only in LOG mode, and never in a load-only run. if ( - self._pipeline_database is not None + not load_only + and self._pipeline_database is not None and self._cache_mode == CacheMode.LOG and self._cached_output_stream is not None ): @@ -1667,6 +1673,7 @@ async def async_execute( output: WritableChannel[tuple[TagProtocol, DataProtocol]], *, observer: "ExecutionObserverProtocol | None" = None, + load_only: bool = False, ) -> None: """Async execution with cache mode handling when DB is attached. @@ -1688,6 +1695,7 @@ async def async_execute( inputs: Sequence of readable channels from upstream nodes. output: Writable channel for output (tag, data) pairs. observer: Optional execution observer for hooks. + load_only: As for ``execute``: a ``LOG`` node stores nothing (ITL-1095). """ from orcapod.pipeline.observer import NoOpObserver @@ -1733,7 +1741,8 @@ async def async_execute( # Delegate to the operator, forward its rows downstream and keep them. intermediate: Channel[tuple[TagProtocol, DataProtocol]] = Channel() should_store = ( - self._pipeline_database is not None + not load_only + and self._pipeline_database is not None and self._cache_mode == CacheMode.LOG ) if should_store: diff --git a/src/orcapod/pipeline/async_orchestrator.py b/src/orcapod/pipeline/async_orchestrator.py index 07f53b7f7..592d6fdb7 100644 --- a/src/orcapod/pipeline/async_orchestrator.py +++ b/src/orcapod/pipeline/async_orchestrator.py @@ -31,6 +31,7 @@ ) if TYPE_CHECKING: + from orcapod.channels import WritableChannel from orcapod.pipeline.dag import GraphProtocol from orcapod.protocols.core_protocols import DataProtocol, TagProtocol from orcapod.protocols.observability_protocols import ExecutionObserverProtocol @@ -75,6 +76,7 @@ def run( materialize_results: bool = True, run_id: str | None = None, pipeline_uri: str = "", + load_only: bool = False, ) -> OrchestratorResult: """Synchronous entry point — runs the async pipeline to completion. @@ -118,6 +120,7 @@ def run( observer=observer, run_id=run_id, pipeline_uri=pipeline_uri, + load_only=load_only, ) ) @@ -138,6 +141,7 @@ def make_run() -> Coroutine[Any, Any, OrchestratorResult]: observer=watched, run_id=run_id, pipeline_uri=pipeline_uri, + load_only=load_only, ) return _run_on_own_thread(make_run, rows_in_flight) @@ -150,6 +154,7 @@ async def run_async( materialize_results: bool = True, run_id: str | None = None, pipeline_uri: str = "", + load_only: bool = False, ) -> OrchestratorResult: """Async entry point for callers already inside an event loop. @@ -171,6 +176,7 @@ async def run_async( observer=observer, run_id=run_id, pipeline_uri=pipeline_uri, + load_only=load_only, ) async def _run_async( @@ -181,6 +187,7 @@ async def _run_async( observer: ExecutionObserverProtocol | None = None, run_id: str | None = None, pipeline_uri: str = "", + load_only: bool = False, ) -> OrchestratorResult: """Core async logic: wire channels, launch tasks, collect results.""" from orcapod.pipeline.observer import NoOpObserver @@ -261,6 +268,7 @@ async def _run_async( writer, observer=effective_observer, run_id=run_id, + load_only=load_only, ), ) elif is_operator_node(node): @@ -280,7 +288,10 @@ async def _run_async( tg, node, node.async_execute( - input_readers, writer, observer=effective_observer + input_readers, + writer, + observer=effective_observer, + load_only=load_only, ), ) elif is_side_effect_node(node): @@ -290,16 +301,20 @@ async def _run_async( "SideEffectNode expects exactly 1 upstream, got 0" ) input_reader = edge_readers[(predecessors[0], node)] - _start_node( - tg, - node, - node.async_execute( - [input_reader], - writer, - observer=effective_observer, - run_id=run_id, - ), - ) + if load_only: + # Nothing is delivered in a load-only run: rows pass on. + _start_node(tg, node, _pass_through(input_reader, writer)) + else: + _start_node( + tg, + node, + node.async_execute( + [input_reader], + writer, + observer=effective_observer, + run_id=run_id, + ), + ) else: raise TypeError( f"Unknown node type: {getattr(node, 'node_type', None)!r}" @@ -595,6 +610,17 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: return NoOpLogger() +async def _pass_through( + reader: Any, writer: WritableChannel[tuple[TagProtocol, DataProtocol]] +) -> None: + """Forward every row of ``reader`` to ``writer``, then close ``writer``.""" + try: + async for item in reader: + await writer.send(item) + finally: + await writer.close() + + def _start_node( tg: asyncio.TaskGroup, node: Any, work: Coroutine[Any, Any, None] ) -> None: diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index e71cf1607..eee77a79a 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -31,13 +31,9 @@ from orcapod.pipeline.base import AbstractPipelineBase from orcapod.pipeline.dag import OrcaDAG from orcapod.pipeline.run_summary import ( - NodeRunSummary, RunCountingObserver, RunSummary, -) -from orcapod.protocols.node_protocols import ( - is_function_node, - is_side_effect_function_node, + run_node_summaries, ) from orcapod.protocols import core_protocols as cp from orcapod.databases.utils import as_table_committing @@ -866,6 +862,8 @@ def run( observer: "ExecutionObserverProtocol | None" = None, materialize_results: bool = True, commit_every_rows: int = 100, + *, + load_only: bool = False, ) -> RunSummary: """Execute the resolvable subgraph of this job in place. @@ -914,6 +912,15 @@ def run( default 1000): the store commits every pending table by itself once one table's pending rows reach that size, which would break the node's results-before-records commit order. + load_only: Fill the nodes from what is already stored and compute + nothing (ITL-1095). Sources are read and operators run on the rows + that reach them; each function node takes the results stored for + this run's inputs (its own records under its content hash) and skips + the inputs without one, which the summary counts (``not_stored``); + a function node forwards only the rows it loaded. No user function + is called, no side effect is delivered (a side-effect node passes its + rows on) and nothing is written to the store, a ``LOG`` operator's + cache included. Afterwards each node shows the rows it loaded. Rows whose function raised (with the default ``error_policy="continue"``) are skipped, and the run goes on. At the end, if any row failed, a single @@ -956,8 +963,9 @@ def run( run_id=prepared.run_id, pipeline_uri=prepared.pipeline_uri, materialize_results=materialize_results, + load_only=load_only, ) - summary = self._finish_run(prepared, counter) + summary = self._finish_run(prepared, counter, load_only=load_only) if not summary.ok: _warn_row_failures(summary, sys._getframe(1)) # pyright: ignore[reportPrivateUsage] # the caller's line is where the warning is shown return summary @@ -968,6 +976,8 @@ async def run_async( observer: "ExecutionObserverProtocol | None" = None, materialize_results: bool = True, commit_every_rows: int = 100, + *, + load_only: bool = False, ) -> RunSummary: """Execute the job with the async orchestrator, from async code. @@ -983,6 +993,7 @@ async def run_async( observer: Optional execution observer. materialize_results: As for ``run``. commit_every_rows: As for ``run``. + load_only: As for ``run``. Returns: The run's ``RunSummary``, as ``run`` returns. @@ -1014,8 +1025,9 @@ async def run_async( run_id=prepared.run_id, pipeline_uri=prepared.pipeline_uri, materialize_results=materialize_results, + load_only=load_only, ) - summary = self._finish_run(prepared, counter) + summary = self._finish_run(prepared, counter, load_only=load_only) if not summary.ok: _warn_row_failures(summary, sys._getframe(1)) # pyright: ignore[reportPrivateUsage] # the caller's line is where the warning is shown return summary @@ -1215,13 +1227,18 @@ def _executing( raise def _finish_run( - self, prepared: "_PreparedRun", counter: RunCountingObserver + self, + prepared: "_PreparedRun", + counter: RunCountingObserver, + *, + load_only: bool = False, ) -> RunSummary: """Commit the run's last writes, mark the job as run and keep its summary. Args: prepared: The run, from ``_prepare_run``. counter: The observer that counted the run's rows. + load_only: Whether the run loaded stored results only. Returns: The run's summary, also kept as ``last_run``. @@ -1255,12 +1272,11 @@ def _finish_run( self._has_run = True self._last_run_finished = True self._run_id = prepared.run_id - node_summaries: list[NodeRunSummary] = [] - for node in prepared.exec_dag.topological_sort(): - label: str = node.label - if is_function_node(node) or is_side_effect_function_node(node): - node_summaries.append(counter.node_summary(label)) - self._last_run = RunSummary("/".join(self._name), node_summaries) + self._last_run = RunSummary( + "/".join(self._name), + run_node_summaries(prepared.exec_dag, counter), + load_only=load_only, + ) return self._last_run def __repr__(self) -> str: diff --git a/src/orcapod/pipeline/run_summary.py b/src/orcapod/pipeline/run_summary.py index cf094c1c4..84124357d 100644 --- a/src/orcapod/pipeline/run_summary.py +++ b/src/orcapod/pipeline/run_summary.py @@ -37,6 +37,11 @@ class NodeRunSummary: failed: Rows whose function raised; the run skipped them. first_error: The exception of the first failed row, with its traceback, or ``None`` when no row failed. + not_stored: Inputs a load-only run skipped because no result is stored for + them (``job.run(load_only=True)``; always 0 in a full run). + inputs_incomplete: The labels of the function nodes upstream of this one that + skipped inputs in a load-only run, in run order: this node's inputs lack + the rows those results would have given (a group may be partial). """ label: str @@ -44,11 +49,13 @@ class NodeRunSummary: reused: int failed: int first_error: BaseException | None + not_stored: int = 0 + inputs_incomplete: tuple[str, ...] = () @property def rows(self) -> int: - """All rows the node received: computed, reused and failed.""" - return self.computed + self.reused + self.failed + """All rows the node received: computed, reused, failed and not stored.""" + return self.computed + self.reused + self.failed + self.not_stored class RunSummary: @@ -61,17 +68,27 @@ class RunSummary: Attributes: job_name: The job's name. nodes: Per function node, in run order, keyed by label. + load_only: Whether the run was a load-only run (``job.run(load_only=True)``): + it computed nothing; ``reused`` counts the rows it loaded. """ - def __init__(self, job_name: str, nodes: Sequence[NodeRunSummary]) -> None: + def __init__( + self, + job_name: str, + nodes: Sequence[NodeRunSummary], + *, + load_only: bool = False, + ) -> None: """Build the summary. Args: job_name: The job's name. nodes: One entry per function node of the run, in run order. + load_only: Whether the run loaded stored results only. """ self.job_name: str = job_name self.nodes: Mapping[str, NodeRunSummary] = {node.label: node for node in nodes} + self.load_only: bool = load_only @property def failed_rows(self) -> int: @@ -121,6 +138,14 @@ def print_traceback(self, label: str | None = None) -> None: def _headline(self) -> str: """The first line: the job and whether any row failed.""" + if self.load_only: + skipped = sum(node.not_stored for node in self.nodes.values()) + outcome = ( + f"{_rows(skipped)} without a stored result skipped." + if skipped + else "every input had a stored result." + ) + return f"Load-only run of job {self.job_name!r}: {outcome}" if self.failed_rows: outcome = f"{_rows(self.failed_rows)} failed." else: @@ -132,8 +157,7 @@ def __repr__(self) -> str: lines = [self._headline()] for node in self.nodes.values(): lines.append( - f" {node.label}: {_rows(node.rows)}: {node.computed} computed, " - f"{node.reused} from store, {node.failed} failed" + f" {node.label}: {_rows(node.rows)}: {_counts(node, self.load_only)}" ) if node.failed: lines.append(f" first error: {_error_line(node.first_error)}") @@ -142,6 +166,8 @@ def __repr__(self) -> str: def _repr_html_(self) -> str: """The notebook display: the same content as ``repr``, as a table.""" + if self.load_only: + return self._load_only_html() header = "".join( f"{name}" for name in ( @@ -177,6 +203,40 @@ def _repr_html_(self) -> str: f"{''.join(body)}" ) + def _load_only_html(self) -> str: + """The notebook display of a load-only run: rows loaded and skipped per node.""" + header = "".join( + f"{name}" + for name in ("node", "rows", "loaded", "not stored", "inputs incomplete") + ) + body = [ + "" + f"{html.escape(node.label)}" + f"{node.rows}{node.reused}{node.not_stored}" + f"{html.escape(', '.join(node.inputs_incomplete))}" + "" + for node in self.nodes.values() + ] + return ( + f"

{html.escape(self._headline())}

" + f"{header}" + f"{''.join(body)}
" + ) + + +def _counts(node: NodeRunSummary, load_only: bool) -> str: + """A node's counts as text: what a full run did, or what a load-only run loaded.""" + if not load_only: + return ( + f"{node.computed} computed, {node.reused} from store, {node.failed} failed" + ) + text = f"{node.reused} loaded" + if node.not_stored: + text += f", {node.not_stored} not stored (skipped)" + if node.inputs_incomplete: + text += f" (inputs incomplete: {', '.join(node.inputs_incomplete)})" + return text + def _rows(n: int) -> str: """``1 row`` or ``n rows``.""" @@ -289,3 +349,51 @@ def on_data_crash( def create_data_logger(self, tag: TagProtocol, data: DataProtocol) -> Any: """No logging of its own.""" return _NOOP_LOGGER + + +def run_node_summaries(dag: Any, counter: RunCountingObserver) -> list[NodeRunSummary]: + """One summary per function node of a finished run, in run order. + + The counts come from ``counter``; a load-only run's skipped inputs come from each + node (``last_run_not_stored``), and every node below a function node that skipped + inputs names it in ``inputs_incomplete`` (ITL-1095). + + Args: + dag: The run's node graph (``topological_sort``, ``predecessors``). + counter: The observer that counted the run's rows. + + Returns: + The summaries of the run's function nodes, in run order. + """ + from dataclasses import replace + + from orcapod.protocols.node_protocols import ( + is_function_node, + is_side_effect_function_node, + ) + + order = list(dag.topological_sort()) + position = {id(node): index for index, node in enumerate(order)} + skipping_above: dict[int, set[int]] = {} + summaries: list[NodeRunSummary] = [] + for node in order: + above: set[int] = set() + for upstream in dag.predecessors(node): + above |= skipping_above.get(id(upstream), set()) + if getattr(upstream, "last_run_not_stored", 0): + above.add(id(upstream)) + skipping_above[id(node)] = above + if not (is_function_node(node) or is_side_effect_function_node(node)): + continue + labels = tuple( + order[index].label for index in sorted(position[key] for key in above) + ) + label: str = getattr(node, "label") # every graph node is labelled + summaries.append( + replace( + counter.node_summary(label), + not_stored=getattr(node, "last_run_not_stored", 0), + inputs_incomplete=labels, + ) + ) + return summaries diff --git a/src/orcapod/pipeline/sync_orchestrator.py b/src/orcapod/pipeline/sync_orchestrator.py index 981706352..af57145b3 100644 --- a/src/orcapod/pipeline/sync_orchestrator.py +++ b/src/orcapod/pipeline/sync_orchestrator.py @@ -55,6 +55,7 @@ def run( materialize_results: bool = True, run_id: str | None = None, pipeline_uri: str = "", + load_only: bool = False, ) -> OrchestratorResult: """Execute the node graph synchronously. @@ -69,6 +70,10 @@ def run( pipeline_uri: Opaque URI string identifying this pipeline run (e.g. ``"my_pipeline@a1b2c3d4e5f6a1b2"``). Forwarded verbatim to ``observer.on_run_start``. + load_only: Fill the nodes from what is stored and compute nothing + (ITL-1095): function nodes load their inputs' stored results and skip + the others, operators run without storing, and a side-effect node + passes its rows on without being called (nothing is delivered). Returns: OrchestratorResult with node outputs. @@ -101,6 +106,7 @@ def run( observer=effective_observer, error_policy=self._error_policy, run_id=run_id, + load_only=load_only, ) elif is_operator_node(node): upstream_buffers = self._gather_upstream_multi(node, graph, buffers) @@ -109,19 +115,23 @@ def run( for buf, upstream_node in upstream_buffers ] buffers[node] = node.execute( - *input_streams, observer=effective_observer + *input_streams, observer=effective_observer, load_only=load_only ) elif is_side_effect_node(node): upstream_buf = self._gather_upstream(node, graph, buffers) - upstream_node = list(graph.predecessors(node))[0] - input_stream = self._materialize_as_stream( - upstream_buf, upstream_node - ) - buffers[node] = node.execute( - input_stream, - observer=effective_observer, - run_id=run_id, - ) + if load_only: + # Nothing is delivered in a load-only run: the rows pass on. + buffers[node] = list(upstream_buf) + else: + upstream_node = list(graph.predecessors(node))[0] + input_stream = self._materialize_as_stream( + upstream_buf, upstream_node + ) + buffers[node] = node.execute( + input_stream, + observer=effective_observer, + run_id=run_id, + ) else: raise TypeError( f"Unknown node type: {getattr(node, 'node_type', None)!r}" diff --git a/src/orcapod/protocols/node_protocols.py b/src/orcapod/protocols/node_protocols.py index 98894e2e8..c33a71c84 100644 --- a/src/orcapod/protocols/node_protocols.py +++ b/src/orcapod/protocols/node_protocols.py @@ -88,6 +88,7 @@ def execute( observer: ExecutionObserverProtocol | None = None, error_policy: Literal["continue", "fail_fast"] = "continue", run_id: str | None = None, + load_only: bool = False, ) -> list[tuple[TagProtocol, DataProtocol]]: ... async def async_execute( @@ -97,6 +98,7 @@ async def async_execute( *, observer: ExecutionObserverProtocol | None = None, run_id: str | None = None, + load_only: bool = False, ) -> None: ... def set_ephemeral_store(self, store: "ArrowDatabaseProtocol | None") -> None: @@ -125,6 +127,7 @@ def execute( self, *input_streams: StreamProtocol, observer: ExecutionObserverProtocol | None = None, + load_only: bool = False, ) -> list[tuple[TagProtocol, DataProtocol]]: ... async def async_execute( @@ -133,6 +136,7 @@ async def async_execute( output: WritableChannel[tuple[TagProtocol, DataProtocol]], *, observer: ExecutionObserverProtocol | None = None, + load_only: bool = False, ) -> None: ... def set_ephemeral_store(self, store: "ArrowDatabaseProtocol | None") -> None: diff --git a/tests/test_core/nodes/test_function_node_empty_data.py b/tests/test_core/nodes/test_function_node_empty_data.py index 7edf9222d..b6ca7555b 100644 --- a/tests/test_core/nodes/test_function_node_empty_data.py +++ b/tests/test_core/nodes/test_function_node_empty_data.py @@ -13,6 +13,7 @@ from orcapod.core.nodes.function_node import FunctionJobNode from orcapod.core.sources import ArrowTableSource from orcapod.databases import InMemoryArrowDatabase +from orcapod.protocols.core_protocols import DataProtocol, TagProtocol from orcapod.system_constants import constants from orcapod.types import NodeConfig from orcapod.errors import EphemeralResultMissingError @@ -174,7 +175,7 @@ def test_normal_result_wins_over_empty_data_token(self, persistent_node): assert isinstance(data, Data) -def _entries(node: FunctionJobNode) -> list: +def _entries(node: FunctionJobNode) -> list[tuple[TagProtocol, DataProtocol]]: """The node's entries for its input rows, as its cache façade returns them. These tests drive ``execute_data`` (one row, not a run), so the node shows no rows diff --git a/tests/test_pipeline/test_job_node_current_run_view.py b/tests/test_pipeline/test_job_node_current_run_view.py index 5b6cf24cd..e9ad8c4b1 100644 --- a/tests/test_pipeline/test_job_node_current_run_view.py +++ b/tests/test_pipeline/test_job_node_current_run_view.py @@ -59,7 +59,7 @@ def _build( src = DictSource(data, tag_columns=["g", "k"], source_id="s") job = PipelineJob(name="j", database=db) with job: - rowwise = inc.pod(src, label="rowwise") + rowwise: Any = inc.pod(src, label="rowwise") grouped = rowwise.group_by("g", label="grouped") total.pod(grouped, label="summed") if cache_mode is not None: From c90eb96662487df4f6ddcee52a55a35aa95a71a8 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:52:55 -0800 Subject: [PATCH 15/63] docs: loading without computing, job.run(load_only=True) (ITL-1095) Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 4 +++- CLAUDE.md | 4 +++- changelog.d/ITL-1095.md | 4 ++++ docs/concepts/function-pods.md | 36 ++++++++++++++++++++++++++++++++++ 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/.zed/rules b/.zed/rules index 6653ea945..f77ed3ea4 100644 --- a/.zed/rules +++ b/.zed/rules @@ -783,7 +783,9 @@ data. The rules: when any function or operator job node it depends on is (core/nodes/staleness.py), and a SourceJobNode is modified when it is bound or its bound source is. A run skips inputs whose entry is already stored. A loaded operator's stored view raises when its run's rows - cannot be named (ITL-1043). + cannot be named (ITL-1043). job.run(load_only=True) computes nothing and writes nothing: function nodes + load their own stored entries for this run's inputs and skip (and count) the others, + operators run without storing, side-effect nodes pass rows on undelivered. - Empty data → ArrowTableSource raises ValueError("Table is empty"). - DerivedSource over a job node with nothing stored → presents zero rows with the origin's columns (no computed records; no error), in the plain form and with tags_as_data. diff --git a/CLAUDE.md b/CLAUDE.md index 66a284382..93ff72c09 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -824,7 +824,9 @@ data. The rules: when any function or operator job node it depends on is (`core/nodes/staleness.py`), and a `SourceJobNode` is modified when it is bound or its bound source is. A run skips inputs whose entry is already stored. A loaded operator's stored view raises when its run's rows - cannot be named (ITL-1043). + cannot be named (ITL-1043). `job.run(load_only=True)` computes nothing and writes nothing: function nodes + load their own stored entries for this run's inputs and skip (and count) the others, + operators run without storing, side-effect nodes pass rows on undelivered. - Empty data → `ArrowTableSource` raises `ValueError("Table is empty")`. - `DerivedSource` over a job node with nothing stored → presents zero rows with the origin's columns (no computed records; no error), in the plain form and with `tags_as_data`. diff --git a/changelog.d/ITL-1095.md b/changelog.d/ITL-1095.md index 0a5c89eeb..8577bb28c 100644 --- a/changelog.d/ITL-1095.md +++ b/changelog.d/ITL-1095.md @@ -4,6 +4,10 @@ - A function or operator job node that has not run in this process is stale (`is_stale` is `True`; it has no modified time), and a job node is stale also when any function or operator job node it depends on is. Binding a source to a job's source slot (`job.bind(sources=...)`) marks the slot modified, so every node fed by it is stale and empty until it runs again. - A loaded operator node no longer raises from `as_table()` or `iter_data()`: it is empty, like every node that has not run. Its stored view (`as_source()`, `get_all_records()`) raises instead when the job file does not record its run's content hash and the store holds nothing under the hash the loaded job computes (ITL-1043). +### Added + +- `job.run(load_only=True)` and `await job.run_async(load_only=True)` fill a job's nodes with what is already stored for this run's inputs and compute nothing (ITL-1095): sources are read, operators run on the rows that reach them (a `CacheMode.LOG` cache writes nothing), each function node takes the stored results of its inputs and skips the inputs without one, a side-effect node passes its rows on without delivering anything, and nothing is written to the store. `RunSummary.load_only`, `NodeRunSummary.not_stored` (inputs skipped) and `NodeRunSummary.inputs_incomplete` (the function nodes above a node that skipped inputs) report it; loaded rows count in `reused`, and the summary's text says "3 loaded, 1 not stored (skipped)". + ### Fixed - On the asynchronous orchestrator, an operator node in `CacheMode.REPLAY` stamps itself after its inputs close, so the rows it replayed are not judged out of date by the time the run ends. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index d32956901..c01237a4f 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -541,6 +541,42 @@ print(summary.ok, summary.failed_rows) # False 3 ``` +### Loading without computing: `run(load_only=True)` + +A job node shows the rows of its current run, so a job built over a store that already holds +its results is empty until it runs. A run that finds every result stored computes nothing, +but it does compute the results that are missing. `job.run(load_only=True)` fills the nodes +with what is already stored for this run's inputs and never computes: sources are read, +operators run on the rows that reach them, each function node takes the stored results of +its inputs and skips the inputs that have none, no side effect is delivered and nothing is +written to the store. The summary counts what each node loaded and skipped: + +```python +more_mice = mice + [{"subject_id": "mouse_04", "weight": 30.0, "height": 0.12}] + +load_job = PipelineJob(name="bmi_demo", database=db) +with load_job: + source = DictSource(data=more_mice, tag_columns=["subject_id"], source_id="mice") + compute_bmi.pod(source, label="bmi") + +print(load_job.nodes["bmi"].as_table().num_rows) # not run in this process: empty +# 0 +calls.clear() +summary = load_job.run(load_only=True) +print(summary) +# Load-only run of job 'bmi_demo': 1 row without a stored result skipped. +# bmi: 4 rows: 3 loaded, 1 not stored (skipped) +print(load_job.nodes["bmi"].as_table().num_rows, "rows; function calls:", len(calls)) +# 3 rows; function calls: 0 +``` + +`summary.nodes["bmi"].not_stored` is the number of skipped inputs, and +`summary.nodes[label].inputs_incomplete` names the function nodes above a node that skipped +inputs. A function node passes on only the rows it loaded, so the rows below it are +incomplete: a group formed after it may lack members, and the result stored for a smaller +group by an earlier run may then be loaded for it. `run_async(load_only=True)` does the same +on the asynchronous orchestrator. + ### When results are committed Inside `job.run()`, each function node commits its results and pipeline records to the From d389828275f8e566ec0af8a6b9fb4b28efc9f0e0 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:56:50 -0800 Subject: [PATCH 16/63] test(pipeline): failing tests: node.run() cascade, state rule and load-only status (ITL-1095) Co-Authored-By: Claude Opus 5.5 --- tests/test_pipeline/test_node_run_cascade.py | 314 +++++++++++++++++++ 1 file changed, 314 insertions(+) create mode 100644 tests/test_pipeline/test_node_run_cascade.py diff --git a/tests/test_pipeline/test_node_run_cascade.py b/tests/test_pipeline/test_node_run_cascade.py new file mode 100644 index 000000000..f986068ab --- /dev/null +++ b/tests/test_pipeline/test_node_run_cascade.py @@ -0,0 +1,314 @@ +"""``node.run()`` runs the job nodes it depends on, by their state (ITL-1095). + +Developer, November 13, 2026: "given that the initially loaded job is empty, I think it's +fair that node.run will cascade node.run on all nodes it depends on recursively. +Critically, node.run(load_only=True) would trigger the upstream node.run also with +load_only=True." And: "if it's not stale, it shouldn't even trigger a run and simply return +what's in the stream cached already." A node filled by a load-only run has its own status +and is never treated as complete by a full run. + +The cascade runs the node's ancestors and the node on the synchronous orchestrator, as +``job.run()`` runs that sub-graph: same rows, same stored records, same counts +(``node.last_run``), and the same end-of-run commit. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any +from unittest import mock + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.core.operators import Join +from orcapod.core.streams import ViewStatus +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.errors import RowFailuresWarning +from orcapod.side_effects import SideEffectPod, tap_pod +from orcapod.sources import DictSource +from orcapod.types import CacheMode + +CALLS: list[str] = [] +DELIVERED: list[int] = [] + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + """Return ``x + 1``.""" + CALLS.append("inc") + return x + 1 + + +@function_pod(output_keys="total") +def total(y: list[int]) -> int: + """Return the sum of a group's values.""" + CALLS.append("total") + return sum(y) + + +@function_pod(output_keys="z") +def dbl(y: int) -> int: + """Return ``2 * y``.""" + CALLS.append("dbl") + return 2 * y + + +@function_pod(output_keys="w") +def neg(y: int) -> int: + """Return ``-y``.""" + CALLS.append("neg") + return -y + + +@function_pod(output_keys="y") +def fails_on_20(x: int) -> int: + """Return ``x + 1``; raise for ``x == 20``.""" + if x == 20: + raise ValueError("x is 20") + return x + 1 + + +def _deliver(y: int, ctx: Any) -> None: + """A side effect: record the delivered value.""" + DELIVERED.append(y) + + +ROWS = [{"g": i % 2, "k": i, "x": 10 * i} for i in range(6)] + + +@pytest.fixture(autouse=True) +def _reset() -> None: + CALLS.clear() + DELIVERED.clear() + + +def _source(data: list[dict[str, int]]) -> DictSource: + return DictSource(data, tag_columns=["g", "k"], source_id="s") + + +def _build( + data: list[dict[str, int]], + db: Any, + *, + cache_mode: CacheMode | None = None, + with_tap: bool = False, +) -> PipelineJob: + """``rowwise`` -> [``tap``] -> ``grouped`` (GroupBy) -> ``summed``.""" + job = PipelineJob(name="j", database=db) + with job: + rows: Any = inc.pod(_source(data), label="rowwise") + if with_tap: + tap = tap_pod(_deliver) + assert isinstance(tap, SideEffectPod) + rows = tap.process(rows, label="tap") + grouped = rows.group_by("g", label="grouped") + total.pod(grouped, label="summed") + if cache_mode is not None: + job.nodes["grouped"].attach_databases( + pipeline_database=db.at("j"), cache_mode=cache_mode + ) + return job + + +def _snapshot(db: Any) -> dict[tuple[str, ...], int]: + db.flush() + counts: dict[tuple[str, ...], int] = {} + for path in db.list_table_paths(): + records = db.get_all_records(path) + counts[path] = 0 if records is None else records.num_rows + return counts + + +def _totals(table: Any) -> list[tuple[int, int]]: + return sorted((r["g"], r["total"]) for r in table.to_pylist()) + + +class TestCascade: + """``node.run()`` first runs every job node it depends on, each once.""" + + def test_a_node_of_a_fresh_job_runs_its_inputs_first(self) -> None: + job = _build(ROWS, InMemoryArrowDatabase()) + view = job.nodes["summed"].run() + assert sorted(CALLS) == ["inc"] * 6 + ["total"] * 2 + assert _totals(view.as_table()) == [(0, 63), (1, 93)] + # Every node of the cascade holds its run, as after job.run(). + for label in ("rowwise", "grouped", "summed"): + assert job.nodes[label].head().status is ViewStatus.ROWS, label + expected = _build(ROWS, InMemoryArrowDatabase()) + expected.run() + for label in ("rowwise", "summed"): + assert ( + job.nodes[label] + .as_table() + .sort_by("g") + .equals(expected.nodes[label].as_table().sort_by("g")) + ), label + + def test_the_cascade_reports_its_counts_on_the_node(self) -> None: + db = InMemoryArrowDatabase() + _build(ROWS[:3], db).run() + CALLS.clear() + job = _build(ROWS, db) + _ = job.nodes["summed"].run() + summary = job.nodes["summed"].last_run + assert summary is not None + assert not summary.load_only + rowwise = summary.nodes["rowwise"] + assert (rowwise.computed, rowwise.reused) == (3, 3) + assert summary.nodes["summed"].computed == 2 + assert "summed" in repr(summary) + + def test_a_diamond_runs_each_ancestor_once(self) -> None: + job = PipelineJob(name="d", database=InMemoryArrowDatabase()) + with job: + top = inc.pod(_source(ROWS), label="top") + left = dbl.pod(top, label="left") + right = neg.pod(top, label="right") + Join()(left, right, label="joined") + view = job.nodes["joined"].run() + assert view.as_table().num_rows == 6 + assert sorted(CALLS) == sorted(["inc"] * 6 + ["dbl"] * 6 + ["neg"] * 6) + + def test_failed_rows_warn_as_a_job_run_does(self) -> None: + job = PipelineJob(name="f", database=InMemoryArrowDatabase()) + with job: + fails_on_20.pod(_source(ROWS), label="node") + with pytest.warns( + RowFailuresWarning, match="1 of 6 rows failed in node 'node'" + ): + view = job.nodes["node"].run() + assert view.as_table().num_rows == 5 + + +class TestState: + """A node that holds a current run is not run again; a stale one is.""" + + def test_a_second_run_of_an_unchanged_node_does_nothing(self) -> None: + db = InMemoryArrowDatabase() + job = _build(ROWS, db) + first = job.nodes["summed"].run() + CALLS.clear() + with ( + mock.patch.object( + InMemoryArrowDatabase, + "get_all_records", + side_effect=AssertionError("store read"), + ), + mock.patch.object( + InMemoryArrowDatabase, + "add_records", + side_effect=AssertionError("store write"), + ), + ): + second = job.nodes["summed"].run() + assert CALLS == [] + assert second.as_table().equals(first.as_table()) + + def test_a_held_ancestor_is_not_run_again(self) -> None: + job = _build(ROWS, InMemoryArrowDatabase()) + _ = job.nodes["rowwise"].run() + stamped = job.nodes["rowwise"].last_modified + CALLS.clear() + _ = job.nodes["summed"].run() + assert job.nodes["rowwise"].last_modified == stamped + assert sorted(CALLS) == ["total", "total"] + + def test_a_node_runs_again_after_its_source_changed(self) -> None: + job = _build(ROWS[:3], InMemoryArrowDatabase()) + _ = job.nodes["summed"].run() + CALLS.clear() + job.bind(sources={next(iter(job.sources)): _source(ROWS)}) + assert job.nodes["summed"].is_stale + view = job.nodes["summed"].run() + assert CALLS.count("inc") == 3 + assert _totals(view.as_table()) == [(0, 63), (1, 93)] + + +class TestLoadOnly: + """``node.run(load_only=True)`` loads through the cascade; its rows have a status.""" + + def test_a_load_only_cascade_computes_and_writes_nothing(self) -> None: + db = InMemoryArrowDatabase() + _build(ROWS[:3], db).run() + CALLS.clear() + before = _snapshot(db) + job = _build(ROWS, db) + view = job.nodes["summed"].run(load_only=True) + assert CALLS == [] + assert _snapshot(db) == before + assert _totals(view.as_table()) == [(0, 22), (1, 11)] + summary = job.nodes["summed"].last_run + assert summary is not None and summary.load_only + assert summary.nodes["rowwise"].not_stored == 3 + rowwise_head = job.nodes["rowwise"].head() + assert rowwise_head.status is ViewStatus.LOADED_ONLY + assert "3 loaded, 3 inputs without a stored result" in rowwise_head.header() + summed_head = job.nodes["summed"].head() + assert summed_head.status is ViewStatus.LOADED_ONLY + assert "inputs incomplete: rowwise" in summed_head.header() + + def test_a_full_run_after_a_load_only_run_computes_what_is_missing(self) -> None: + db = InMemoryArrowDatabase() + _build(ROWS[:3], db).run() + CALLS.clear() + job = _build(ROWS, db) + _ = job.nodes["summed"].run(load_only=True) + assert CALLS == [] + view = job.nodes["summed"].run() + assert CALLS.count("inc") == 3 + assert _totals(view.as_table()) == [(0, 63), (1, 93)] + assert job.nodes["rowwise"].head().status is ViewStatus.ROWS + + def test_a_load_only_run_of_a_load_only_node_returns_what_it_holds(self) -> None: + db = InMemoryArrowDatabase() + _build(ROWS[:3], db).run() + job = _build(ROWS, db) + first = job.nodes["summed"].run(load_only=True) + stamped = job.nodes["rowwise"].last_modified + second = job.nodes["summed"].run(load_only=True) + assert job.nodes["rowwise"].last_modified == stamped + assert second.as_table().equals(first.as_table()) + + def test_a_load_only_cascade_uses_a_fully_run_ancestor(self) -> None: + job = _build(ROWS, InMemoryArrowDatabase()) + _ = job.nodes["rowwise"].run() + stamped = job.nodes["rowwise"].last_modified + _ = job.nodes["summed"].run(load_only=True) + assert job.nodes["rowwise"].last_modified == stamped + + def test_a_load_only_run_that_found_everything_stays_loaded_only(self) -> None: + db = InMemoryArrowDatabase() + _build(ROWS, db).run() + job = _build(ROWS, db) + _ = job.nodes["rowwise"].run(load_only=True) + head = job.nodes["rowwise"].head() + assert head.status is ViewStatus.LOADED_ONLY + assert "6 loaded, 0 inputs without a stored result" in head.header() + + +class TestSideEffects: + def test_a_full_cascade_delivers_a_load_only_cascade_does_not(self) -> None: + db = InMemoryArrowDatabase() + job = _build(ROWS, db, with_tap=True) + _ = job.nodes["summed"].run(load_only=True) + assert DELIVERED == [] + _ = job.nodes["summed"].run() + assert sorted(DELIVERED) == sorted(r["x"] + 1 for r in ROWS) + + +def test_the_cascade_commits_every_write(tmp_path: Path) -> None: + """A store opened again through a new object holds every result and record.""" + path = tmp_path / "store" + job = _build(ROWS, DeltaTableDatabase(path), cache_mode=CacheMode.LOG) + _ = job.nodes["summed"].run() + CALLS.clear() + probe = _build(ROWS, DeltaTableDatabase(path), cache_mode=CacheMode.LOG) + for label, rows in (("rowwise", 6), ("grouped", 2), ("summed", 2)): + records = probe.nodes[label].get_all_records() + assert records is not None and records.num_rows == rows, label + again = _build(ROWS, DeltaTableDatabase(path)) + summary = again.run() + assert CALLS == [] + assert summary.nodes["rowwise"].reused == 6 + assert summary.nodes["summed"].reused == 2 From db1b80af02a809fef323d700f04d0fb8de732a44 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 22:01:20 -0800 Subject: [PATCH 17/63] feat(nodes): node.run() runs the job nodes it depends on, by their state (ITL-1095) A job node's run(*, load_only=False) is state-driven: a node that holds a completed run and is not stale (and, for a full run, was not filled load-only, nor anything above it) returns its rows and runs nothing. Otherwise it runs, after every job node it depends on that holds no current run, each once, on the sync orchestrator restricted to that sub-graph (held rows feed the nodes after them), and commits the run's writes as job.run() does. node.last_run reports the counts; failed rows warn. A load-only run's rows have their own status, ViewStatus.LOADED_ONLY, captioned 'N loaded, M inputs without a stored result' (and the incomplete inputs above). CACHE_ONLY nodes and nodes that cannot compute raise from run() unless load_only. Co-Authored-By: Claude Opus 5.5 --- docs/concepts/function-pods.md | 3 +- src/orcapod/core/nodes/function_node.py | 175 ++++++++++++++++++---- src/orcapod/core/nodes/operator_node.py | 73 +++++++-- src/orcapod/core/nodes/source_node.py | 14 ++ src/orcapod/core/nodes/staleness.py | 43 ++++++ src/orcapod/core/streams/view.py | 14 +- src/orcapod/pipeline/job.py | 31 +--- src/orcapod/pipeline/node_run.py | 145 ++++++++++++++++++ src/orcapod/pipeline/run_summary.py | 71 +++++++-- src/orcapod/pipeline/sync_orchestrator.py | 12 +- tests/test_core/streams/test_run.py | 17 ++- 11 files changed, 514 insertions(+), 84 deletions(-) create mode 100644 src/orcapod/pipeline/node_run.py diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index c01237a4f..cbb4a4415 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -345,7 +345,8 @@ result is read from the store and nothing is computed. A run that raised defines and so does `clear_cache()`; a node whose input changed since its run (an upstream node ran again, or a source was bound again) is stale and empty until it runs again. The stored view, `node.as_source()`, reads every record stored under the node's key, whether or not the node -has run. +has run: the node's stored history, which keeps an entry's earlier result next to its +recomputed one, where a run loads only the latest. `InMemoryArrowDatabase` lasts only as long as the Python process. Use `DeltaTableDatabase("some/directory")` from `orcapod.databases` to keep results on disk diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index ff83b57bf..61250eb25 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -18,7 +18,9 @@ import asyncio import contextlib +import dataclasses import logging +import sys import uuid from abc import abstractmethod from collections import Counter @@ -1574,6 +1576,12 @@ def _set_run_rows( self._cached_output_table = None self._cached_content_hash_column = None + def _holds_run(self, *, load_only: bool) -> bool: + """Whether the node holds a completed run; a load-only one only for ``load_only``.""" + return self._run_entry_counts is not None and ( + load_only or not self._run_load_only + ) + @property def last_run_not_stored(self) -> int: """Inputs the node's last completed run skipped because nothing was stored. @@ -3340,7 +3348,15 @@ def head( return self._head_without_rows( config, ViewStatus.NOT_COMPUTED, note=STALE_NOTE ) - return self._head_of_pairs(list(self._held_pairs()), n, config, origin="held") + head = self._head_of_pairs(list(self._held_pairs()), n, config, origin="held") + if not self._run_load_only: + return head + # A load-only run's rows have their own status: they may lack the inputs it + # skipped (ITL-1095). + note = self._load_only_note() + if head.note: + note += f"; {head.note}" + return dataclasses.replace(head, status=ViewStatus.LOADED_ONLY, note=note) # ------------------------------------------------------------------ # Iteration @@ -3393,44 +3409,149 @@ def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: self.clear_cache() yield from self._held_pairs() - def run(self) -> StreamView: - """Eagerly compute all input data, filling pipeline and result databases, and - return a ``StreamView`` of the node's rows. + def run(self, *, load_only: bool = False) -> StreamView: + """Run the node as a job run would, and return a ``StreamView`` of its rows. - Computes as a job run computes this node, unlike ``as_table()``, which only - reads. A node whose upstream is unavailable (``CACHE_ONLY``) or whose function - pod is a placeholder (``READ_ONLY``) computes nothing and holds no rows; its - stored rows are read with ``as_source()``. + State-driven (developer, November 13, 2026; ITL-1095): a node that holds the rows + of a completed run and is not stale is not run again; the view shows what it + holds, and nothing is computed, read or written. Otherwise the node runs, after + every job node it depends on that holds no current run (each once, in + dependency order, as ``job.run()`` would run them; ``last_run`` reports the + counts), and the run's writes are committed. A node built over streams outside a + job reads its input as it is. + + Args: + load_only: Fill the node, and the nodes it runs first, with what is already + stored and compute nothing (as ``job.run(load_only=True)``). A node + filled this way shows status ``loaded_only`` and is not current for a + later full ``run()``, which computes what it skipped. Returns: - A ``StreamView`` of every row ``as_table()`` then returns. It says "rows", - as ``head()`` then does: the node holds them in memory, whether it computed - them or read them from its store. + A ``StreamView`` of every row ``as_table()`` then returns: status ``rows``, + or ``loaded_only`` after a load-only run. Raises: - RuntimeError: If the node was loaded without a function pod, or - ``load_status`` is UNAVAILABLE (no pod, no DB). - """ + RuntimeError: If no store is attached (``UNAVAILABLE``); if the node cannot + compute (loaded without a function pod, or a placeholder pod) and + ``load_only`` is false; or if its input is unavailable + (``CACHE_ONLY``): read its stored rows with ``as_source()``. + """ + from orcapod.pipeline.node_run import ( + depends_on_job_nodes, + run_with_dependencies, + ) + from orcapod.pipeline.run_summary import warn_row_failures from orcapod.pipeline.serialization import LoadStatus - if self._function_pod is None: - raise self._cannot_compute_error() - self._begin_run() - if self._load_status == LoadStatus.UNAVAILABLE: + status = self._load_status + if status == LoadStatus.UNAVAILABLE: raise self._no_store_error() - if self._load_status in (LoadStatus.CACHE_ONLY, LoadStatus.READ_ONLY): - # CACHE_ONLY: upstream unavailable; computation requires a live input stream. - # READ_ONLY: function pod is a proxy placeholder — cannot compute. - # Neither runs, so the view holds no rows (ITL-1095). - return self._view_of_rows(self.as_table(), "held") + if status == LoadStatus.CACHE_ONLY: + raise RuntimeError( + f"FunctionJobNode {self.label!r}: its input is unavailable, so it has " + "no inputs to run or look up; read its stored results with as_source()." + ) + if not load_only and ( + self._function_pod is None or status == LoadStatus.READ_ONLY + ): + raise self._cannot_compute_error() + if not self.holds_current_run(load_only=load_only): + if depends_on_job_nodes(self): + summary = run_with_dependencies(self, load_only=load_only) + else: + summary = self._run_alone(load_only=load_only) + self._last_run = summary + if not summary.ok: + warn_row_failures(summary, sys._getframe(1)) # pyright: ignore[reportPrivateUsage] # the caller's line is where the warning is shown + return self._run_view() + + def _run_alone(self, *, load_only: bool) -> Any: + """Run this node over its input stream, which is not a job node; commit. + + Returns: + The run's ``RunSummary``. + """ + from dataclasses import replace + + from orcapod.pipeline.run_summary import RunCountingObserver, RunSummary + from orcapod.utils.store_errors import flush_best_effort + if self.is_stale: # Discard any stale in-memory entries before a fresh computation run # so that rerunning does not mix old cached entries with new results. self.clear_cache() - self.execute(self._input_stream) - # The rows are the ones held in memory, as head() names them afterwards: some - # may have been read from the result store rather than computed. - return self._view_of_rows(self.as_table(), "held") + counter = RunCountingObserver() + stores = [ + store + for store in ( + self._pipeline_database, + getattr(self._cached_function_pod, "result_database", None), + ) + if store is not None + ] + try: + _ = self.execute(self._input_stream, observer=counter, load_only=load_only) + except Exception as run_error: + flush_best_effort(run_error, *stores) + raise + for store in stores: + store.flush() + node_summary = replace( + counter.node_summary(self.label), not_stored=self.last_run_not_stored + ) + base_path = getattr(self._pipeline_database, "base_path", ()) + return RunSummary( + "/".join(base_path), [node_summary], load_only=load_only, target=self.label + ) + + def _incomplete_inputs(self) -> list[str]: + """Labels of the function nodes above this one whose load-only run skipped inputs.""" + labels: list[str] = [] + seen: list[Any] = [] + pending: list[Any] = list(self.upstreams) + while pending: + node = pending.pop() + if any(node is other for other in seen): + continue + seen.append(node) + if isinstance(node, FunctionJobNode) and node._run_load_only: + if node._run_not_stored: + labels.append(node.label) + pending.extend(getattr(node, "upstreams", ())) + return sorted(labels) + + def _load_only_note(self) -> str: + """The caption of a node filled by a load-only run (ITL-1095).""" + note = ( + f"loaded only: {self._run_loaded_count():,} loaded, " + f"{self._run_not_stored:,} inputs without a stored result" + ) + incomplete = self._incomplete_inputs() + if incomplete: + note += f"; inputs incomplete: {', '.join(incomplete)}" + return note + + def _run_loaded_count(self) -> int: + """How many inputs of the last run had a stored result (loaded).""" + counts = self._run_entry_counts + return 0 if counts is None else sum(counts.values()) + + def _run_view(self) -> StreamView: + """The ``run()`` view of the node's rows, with the status of its last run.""" + table = self.as_table() + if not self._run_load_only: + # The rows are the ones held in memory, as head() names them afterwards: + # some may have been read from the result store rather than computed. + return self._view_of_rows(table, "held") + return StreamView.of_rows( + self, + table, + tag_columns=self.keys()[0], + total_rows=table.num_rows, + status=ViewStatus.LOADED_ONLY, + note=self._load_only_note(), + display_all_rows=False, + ) # ------------------------------------------------------------------ # Async channel execution diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 16c3321d6..a9aa8ec30 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -17,6 +17,7 @@ from __future__ import annotations import asyncio +import sys import logging from abc import abstractmethod from collections.abc import Collection, Iterator, Mapping, Sequence @@ -761,6 +762,7 @@ def clear_cache(self) -> None: super().clear_cache() # clears _node_identity_path_cache + _update_modified_time self._cached_output_stream = None self._cached_output_table = None + self._run_load_only = False # No content established: the node, and every node downstream, is stale. self._set_modified_time(None) @@ -1350,6 +1352,7 @@ def execute( ctx_obs.on_node_start(node_label, node_hash) + self._run_load_only = load_only # Check REPLAY cache first cached_output = self.get_cached_output() if cached_output is not None: @@ -1385,13 +1388,17 @@ def execute( ctx_obs.on_node_end(node_label, node_hash) return output - def _compute_and_store(self) -> None: - """Compute operator output, optionally store in DB.""" + def _compute_and_store(self, *, store: bool = True) -> None: + """Compute operator output, optionally store in DB. + + Args: + store: Whether a ``LOG`` node stores the output (not in a load-only run). + """ self._cached_output_stream = self._operator.process( *self._input_streams, ) - if self._cache_mode == CacheMode.OFF: + if self._cache_mode == CacheMode.OFF or not store: self._update_modified_time() return @@ -1432,12 +1439,15 @@ def _replay_from_cache(self) -> None: ) self._update_modified_time() - def run(self) -> StreamView: - """Execute the operator according to the current cache mode, and return a - ``StreamView`` of the node's rows. + def run(self, *, load_only: bool = False) -> StreamView: + """Run the node as a job run would, and return a ``StreamView`` of its rows. - Computes as a job run computes this node, unlike ``as_table()``, which only - reads. + State-driven (developer, November 13, 2026; ITL-1095): a node that holds the + output of a completed run and is not stale is not run again; the view shows it. + Otherwise the node runs, after every job node it depends on that holds no + current run (each once, in dependency order, as ``job.run()`` would run them; + ``last_run`` reports the counts), and the run's writes are committed. A node + built over streams outside a job runs its operator on them directly. Without a database: Always compute via the operator's ``process()`` method. @@ -1447,10 +1457,35 @@ def run(self) -> StreamView: - **LOG**: always compute, write results to DB. - **REPLAY**: skip computation, load from DB. + Args: + load_only: As ``job.run(load_only=True)``: the operator runs on the rows + that reach it, a ``LOG`` node writes nothing, and the nodes run first + load their stored results only. + Returns: A ``StreamView`` of every row ``as_table()`` then returns, naming its rows as ``head()`` then names them. + + Raises: + RuntimeError: If the node was loaded by ``PipelineJob.load`` and + ``load_only`` is false. """ + from orcapod.pipeline.node_run import ( + depends_on_job_nodes, + run_with_dependencies, + ) + from orcapod.pipeline.run_summary import warn_row_failures + + if self._loaded and not load_only: + raise self._cannot_compute_error() + if self.holds_current_run(load_only=load_only): + return self._run_view(computed=False) + if depends_on_job_nodes(self): + summary = run_with_dependencies(self, load_only=load_only) + self._last_run = summary + if not summary.ok: + warn_row_failures(summary, sys._getframe(1)) # pyright: ignore[reportPrivateUsage] # the caller's line is where the warning is shown + return self._run_view(computed=False) if self._loaded: raise self._cannot_compute_error() if self.is_stale: @@ -1462,14 +1497,23 @@ def run(self) -> StreamView: if self._cache_mode == CacheMode.REPLAY: self._replay_from_cache() else: - self._compute_and_store() + self._compute_and_store(store=not load_only) computed = True + self._run_load_only = load_only else: self._cached_output_stream = self._operator.process( *self._input_streams, ) self._update_modified_time() computed = True + return self._run_view(computed=computed) + + def _run_view(self, *, computed: bool) -> StreamView: + """The ``run()`` view of the node's output. + + Args: + computed: Whether this ``run()`` computed the output lazily just now. + """ # Read the kept output itself: a lazy input that finishes computing during this # read stamps itself after the node, which would make as_table() judge it stale. assert self._cached_output_stream is not None @@ -1486,6 +1530,16 @@ def run(self) -> StreamView: origin = "computed" if kept is None else head_of(kept.head(1), self).origin return self._view_of_rows(table, origin) + # Whether the output the node holds came from a load-only run (ITL-1095): its + # input may lack rows, so a full run does not take it as complete. + _run_load_only: bool = False + + def _holds_run(self, *, load_only: bool) -> bool: + """Whether the node holds a run's output; a load-only run's only for ``load_only``.""" + return self._cached_output_stream is not None and ( + load_only or not self._run_load_only + ) + def _current_output(self) -> StreamProtocol | None: """The output of the node's current run, or ``None`` when it holds none. @@ -1710,6 +1764,7 @@ async def async_execute( try: ctx_obs.on_node_start(node_label, node_hash) + self._run_load_only = load_only if ( self._pipeline_database is not None diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index a7535d2b1..07537c705 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -591,6 +591,20 @@ def __init__( # bound_source property — explicit binding with cache invalidation # ------------------------------------------------------------------ + def run(self, *, load_only: bool = False) -> StreamView: + """Read the bound source's rows and return a ``StreamView`` of all of them. + + A source is read, never computed, so ``load_only`` changes nothing; a job + node's ``run()`` takes it (ITL-1095). + + Args: + load_only: Accepted for every job node; a source reads its rows either way. + + Raises: + UnboundSourceError: When no concrete source is bound. + """ + return super().run() + @property def bound_source(self) -> StreamProtocol | None: """The concrete stream currently bound to this slot, or ``None``.""" diff --git a/src/orcapod/core/nodes/staleness.py b/src/orcapod/core/nodes/staleness.py index 06efbe5b8..841a96fd0 100644 --- a/src/orcapod/core/nodes/staleness.py +++ b/src/orcapod/core/nodes/staleness.py @@ -20,6 +20,20 @@ class RunHoldingNode: their own ``is_stale`` is redefined by ITL-1091. """ + # The summary of the node's last ``run()`` (with the nodes it ran first); a class + # default so nodes rebuilt from a saved descriptor start without one. + _last_run: Any = None + + @property + def last_run(self) -> Any: + """The ``RunSummary`` of this node's last ``run()``, or ``None``. + + ``node.run()`` runs the node and the job nodes it depends on (ITL-1095); this is + what that run did, per function node, as ``job.last_run`` is for ``job.run()``. + A ``run()`` that found the node current ran nothing and leaves it unchanged. + """ + return self._last_run + @property def is_stale(self) -> bool: """Whether the rows this node holds are out of date, or it holds none. @@ -32,6 +46,35 @@ def is_stale(self) -> bool: own: bool = super().is_stale # pyright: ignore[reportAttributeAccessIssue] # the stream base after this mixin in the MRO defines it return own or upstream_job_node_stale(self.upstreams) # pyright: ignore[reportAttributeAccessIssue] # every node class this is mixed into has upstreams + def holds_current_run(self, *, load_only: bool = False) -> bool: + """Whether the node holds the rows of a run that ``run()`` would not redo. + + A node that is not stale and holds the rows of a completed run in this process + is current: ``run()`` returns them without running it (developer, November 13, + 2026, ITL-1095). A node filled by a load-only run is current only for another + load-only run: a full run computes the inputs it skipped. So is every node + whose rows came from such a node: a node is current only if every job node + above it is current for the same kind of run. + + Args: + load_only: Whether the question is for a load-only run. + + Returns: + ``True`` if the node holds a current run for such a run. + """ + if not self._holds_run(load_only=load_only) or self.is_stale: + return False + upstreams: tuple[Any, ...] = getattr(self, "upstreams", ()) + return all( + upstream.holds_current_run(load_only=load_only) + for upstream in upstreams + if isinstance(upstream, RunHoldingNode) + ) + + def _holds_run(self, *, load_only: bool) -> bool: + """Whether the node holds the rows of a completed run good for such a run.""" + raise NotImplementedError + def upstream_job_node_stale(upstreams: Iterable[Any]) -> bool: """Return whether any job node in ``upstreams`` that holds run rows is stale. diff --git a/src/orcapod/core/streams/view.py b/src/orcapod/core/streams/view.py index f440bde36..d81e93c16 100644 --- a/src/orcapod/core/streams/view.py +++ b/src/orcapod/core/streams/view.py @@ -45,6 +45,9 @@ class ViewStatus(str, Enum): ROWS: The rows are the first rows of the stream's complete content. Zero rows means the content is empty. PARTIAL: A computation stopped part way; the rows are those computed so far. + LOADED_ONLY: A job node filled by a load-only run (``run(load_only=True)``): + the rows are the stored results of its inputs; inputs without one were + skipped, so the rows may be fewer than a full run gives (ITL-1095). NOT_COMPUTED: Nothing is computed, or what was computed is out of date. NOTHING_STORED: A stored view (``node.as_source()``) whose store holds no rows for its node yet. @@ -54,6 +57,7 @@ class ViewStatus(str, Enum): ROWS = "rows" PARTIAL = "partial" + LOADED_ONLY = "loaded_only" NOT_COMPUTED = "not_computed" NOTHING_STORED = "nothing_stored" UNAVAILABLE = "unavailable" @@ -372,7 +376,7 @@ def _is_window(self) -> bool: can be named by position: a ``ROWS`` view with no ``description``, unsorted or sorted among all of the stream's rows.""" return ( - self.status is ViewStatus.ROWS + self.status in (ViewStatus.ROWS, ViewStatus.LOADED_ONLY) and self.description is None and (not self.sorted_by_tags or self._total_is_exact()) ) @@ -392,7 +396,11 @@ def _holds_every_row(self) -> bool: def _cuttable(self) -> StreamView: """Return this view in a form whose rows can be cut by position: itself when its rows are a run of the stream's rows (or of a pool), else a pool of its rows.""" - if self.status not in (ViewStatus.ROWS, ViewStatus.PARTIAL): + if self.status not in ( + ViewStatus.ROWS, + ViewStatus.PARTIAL, + ViewStatus.LOADED_ONLY, + ): return self if self.pool is not None or self._is_window(): return self @@ -423,7 +431,7 @@ def _describe(self) -> str: shown = self.table.num_rows if self.pool is not None: return _window_text(self.pool, shown, self.start, self.pool_rows) - if self.status is ViewStatus.ROWS: + if self.status in (ViewStatus.ROWS, ViewStatus.LOADED_ONLY): text = self._describe_rows(shown) elif self.status is ViewStatus.PARTIAL: total = self.total_rows if self.total_rows is not None else shown diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index eee77a79a..30cd42bbe 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -5,7 +5,6 @@ import contextlib import logging import sys -import warnings from collections.abc import Iterator from dataclasses import dataclass from typing import TYPE_CHECKING, Any @@ -22,7 +21,6 @@ unclaimed_tables_written_before_v03, ) from orcapod.errors import ( - RowFailuresWarning, StoreError, StoreReadError, StoreWriteError, @@ -34,6 +32,7 @@ RunCountingObserver, RunSummary, run_node_summaries, + warn_row_failures, ) from orcapod.protocols import core_protocols as cp from orcapod.databases.utils import as_table_committing @@ -42,7 +41,6 @@ if TYPE_CHECKING: from pathlib import Path - from types import FrameType from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator from orcapod.pipeline.execution_context import ExecutionContext @@ -98,29 +96,6 @@ def _with_counter( return CompositeObserver(observer, counter) # pyright: ignore[reportReturnType] # forwards every hook; its **kwargs signatures do not match the protocol structurally -def _warn_row_failures(summary: RunSummary, caller: FrameType) -> None: - """Issue the run's ``RowFailuresWarning``, shown after every run with failed rows. - - Python's default filter shows a warning once per text and line, so a second identical - failing run from the same line (re-running a notebook cell) would show nothing. A - fresh registry for each run keeps every run's warning, while the user's filters - (``ignore``, ``error``) still apply. - - Args: - summary: The run's summary, with failed rows. - caller: The frame that called ``run``; the warning is shown at its line. - """ - warnings.warn_explicit( - summary.failures_message(), - RowFailuresWarning, - caller.f_code.co_filename, - caller.f_lineno, - module=caller.f_globals.get("__name__"), - registry=None, - module_globals=caller.f_globals, - ) - - class PipelineJob(AbstractPipelineBase[JobNode]): """Pipeline + source bindings + execution context. @@ -967,7 +942,7 @@ def run( ) summary = self._finish_run(prepared, counter, load_only=load_only) if not summary.ok: - _warn_row_failures(summary, sys._getframe(1)) # pyright: ignore[reportPrivateUsage] # the caller's line is where the warning is shown + warn_row_failures(summary, sys._getframe(1)) # pyright: ignore[reportPrivateUsage] # the caller's line is where the warning is shown return summary async def run_async( @@ -1029,7 +1004,7 @@ async def run_async( ) summary = self._finish_run(prepared, counter, load_only=load_only) if not summary.ok: - _warn_row_failures(summary, sys._getframe(1)) # pyright: ignore[reportPrivateUsage] # the caller's line is where the warning is shown + warn_row_failures(summary, sys._getframe(1)) # pyright: ignore[reportPrivateUsage] # the caller's line is where the warning is shown return summary def _prepare_run(self, commit_every_rows: int) -> "_PreparedRun": diff --git a/src/orcapod/pipeline/node_run.py b/src/orcapod/pipeline/node_run.py new file mode 100644 index 000000000..3cdf8d468 --- /dev/null +++ b/src/orcapod/pipeline/node_run.py @@ -0,0 +1,145 @@ +"""``node.run()`` of a job node: run it and the job nodes it depends on (ITL-1095). + +Developer, November 13, 2026: "given that the initially loaded job is empty, I think it's +fair that node.run will cascade node.run on all nodes it depends on recursively. +Critically, node.run(load_only=True) would trigger the upstream node.run also with +load_only=True." And: "if it's not stale, it shouldn't even trigger a run and simply +return what's in the stream cached already." + +The cascade is ``job.run()`` restricted to the node's dependencies, on the synchronous +orchestrator: a dependency that holds a current run is not run again (its rows feed the +nodes after it), every other one runs once, in dependency order, and the run ends with +the commit ``job.run()`` makes. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from orcapod.errors import StoreError, StoreWriteError +from orcapod.pipeline.run_summary import ( + RunCountingObserver, + RunSummary, + run_node_summaries, +) +from orcapod.utils.store_errors import flush_best_effort + +if TYPE_CHECKING: + from orcapod.protocols.database_protocols import ArrowDatabaseProtocol + + +def _is_job_node(stream: Any) -> bool: + """Whether ``stream`` is a node of a job (as opposed to a stream outside any job).""" + from orcapod.core.nodes.source_node import SourceJobNode + from orcapod.core.nodes.staleness import RunHoldingNode + from orcapod.side_effects import SideEffectJobNode + + return isinstance(stream, (RunHoldingNode, SideEffectJobNode, SourceJobNode)) + + +def depends_on_job_nodes(node: Any) -> bool: + """Whether any direct input of ``node`` is a job node. + + A node built directly over streams outside a job (a source, a pod's stream) has + nothing to run first: its ``run()`` reads its input as it always has. + """ + return any(_is_job_node(upstream) for upstream in node.upstreams) + + +def _stores(nodes: list[Any]) -> list[ArrowDatabaseProtocol]: + """The distinct stores the given nodes write to: pipeline and result databases.""" + found: list[ArrowDatabaseProtocol] = [] + for node in nodes: + pipeline_database = getattr(node, "_pipeline_database", None) + cached_pod = getattr(node, "_cached_function_pod", None) + result_database = getattr(cached_pod, "result_database", None) + for database in (pipeline_database, result_database): + if database is not None and not any(database is d for d in found): + found.append(database) + return found + + +def _job_name(node: Any) -> str: + """The name of the job ``node`` belongs to: its pipeline database's location.""" + database = getattr(node, "_pipeline_database", None) + base_path: tuple[str, ...] = getattr(database, "base_path", ()) + return "/".join(base_path) + + +def run_with_dependencies(target: Any, *, load_only: bool) -> RunSummary: + """Run ``target`` and every job node it depends on that holds no current run. + + The dependencies are found through ``upstreams``. A dependency that holds a current + run (``holds_current_run``) is not run: its rows feed the nodes after it. A stream + outside the job is read. Every other node runs once, in dependency order, on the + ``SyncPipelineOrchestrator`` (error policy ``"continue"``), as ``job.run()`` would + run that sub-graph; with ``load_only`` as ``job.run(load_only=True)`` would. The run + ends with a commit of every store it wrote to, also, best-effort, when it fails. + + Args: + target: The job node whose ``run()`` this is. + load_only: Whether to load stored results only. + + Returns: + The run's summary, one entry per function node that ran. + + Raises: + StoreWriteError: If committing the run's writes failed. + """ + from orcapod.core.nodes.source_node import SourceJobNode + from orcapod.core.nodes.staleness import RunHoldingNode + from orcapod.pipeline.dag import OrcaDAG + from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator + + dag: OrcaDAG[Any] = OrcaDAG() + held: dict[Any, list[Any]] = {} + seen: list[Any] = [] + + def visit(node: Any) -> None: + """Add ``node`` and, unless it is held or a source, what it depends on.""" + if any(node is other for other in seen): + return + seen.append(node) + dag.add_node(node) + if ( + node is not target + and isinstance(node, RunHoldingNode) + and node.holds_current_run(load_only=load_only) + ): + held[node] = list(node.iter_data()) + return + if isinstance(node, SourceJobNode): + return + if not _is_job_node(node): + # A stream outside the job: its rows, read as the node's run reads them. + held[node] = list(node.iter_data()) + return + for upstream in node.upstreams: + visit(upstream) + dag.add_edge(upstream, node) + + visit(target) + run_nodes = [node for node in seen if not any(node is h for h in held)] + stores = _stores(run_nodes) + counter = RunCountingObserver() + try: + _ = SyncPipelineOrchestrator().run( + dag, observer=counter, load_only=load_only, held_rows=held + ) + except Exception as run_error: + # Commit the writes that succeeded before the failure, as job.run() does. + flush_best_effort(run_error, *stores) + raise + for store in stores: + try: + store.flush() + except StoreError: + raise + except Exception as exc: + raise StoreWriteError(store.base_path, flush=True) from exc + return RunSummary( + _job_name(target), + run_node_summaries(dag, counter, skip=list(held)), + load_only=load_only, + target=target.label, + ) diff --git a/src/orcapod/pipeline/run_summary.py b/src/orcapod/pipeline/run_summary.py index 84124357d..ee48b49f3 100644 --- a/src/orcapod/pipeline/run_summary.py +++ b/src/orcapod/pipeline/run_summary.py @@ -18,7 +18,8 @@ from orcapod.pipeline.observer import NoOpLogger if TYPE_CHECKING: - from collections.abc import Mapping, Sequence + from collections.abc import Collection, Mapping, Sequence + from types import FrameType from orcapod.protocols.core_protocols import DataProtocol, TagProtocol from orcapod.types import SchemaLike @@ -70,6 +71,8 @@ class RunSummary: nodes: Per function node, in run order, keyed by label. load_only: Whether the run was a load-only run (``job.run(load_only=True)``): it computed nothing; ``reused`` counts the rows it loaded. + target: For the run of one node and the nodes it depends on + (``node.run()``), that node's label; ``None`` for a job run. """ def __init__( @@ -78,6 +81,7 @@ def __init__( nodes: Sequence[NodeRunSummary], *, load_only: bool = False, + target: str | None = None, ) -> None: """Build the summary. @@ -85,10 +89,12 @@ def __init__( job_name: The job's name. nodes: One entry per function node of the run, in run order. load_only: Whether the run loaded stored results only. + target: The label of the node whose ``run()`` this was, if any. """ self.job_name: str = job_name self.nodes: Mapping[str, NodeRunSummary] = {node.label: node for node in nodes} self.load_only: bool = load_only + self.target: str | None = target @property def failed_rows(self) -> int: @@ -109,7 +115,7 @@ def failures_message(self) -> str: return " ".join( f"{node.failed} of {_rows(node.rows)} failed in node {node.label!r} " f"(first error: {_error_line(node.first_error)}). " - f"Full traceback: {_traceback_call(node.label)}." + f"Full traceback: {_traceback_call(node.label, self.target)}." for node in self.nodes.values() if node.failed ) @@ -137,7 +143,12 @@ def print_traceback(self, label: str | None = None) -> None: print(f"No rows failed{where} in this run.") def _headline(self) -> str: - """The first line: the job and whether any row failed.""" + """The first line: the job (or node) and whether any row failed.""" + subject = ( + f"job {self.job_name!r}" + if self.target is None + else f"node {self.target!r} and the nodes it depends on" + ) if self.load_only: skipped = sum(node.not_stored for node in self.nodes.values()) outcome = ( @@ -145,12 +156,12 @@ def _headline(self) -> str: if skipped else "every input had a stored result." ) - return f"Load-only run of job {self.job_name!r}: {outcome}" + return f"Load-only run of {subject}: {outcome}" if self.failed_rows: outcome = f"{_rows(self.failed_rows)} failed." else: outcome = "all rows succeeded." - return f"Run of job {self.job_name!r}: {outcome}" + return f"Run of {subject}: {outcome}" def __repr__(self) -> str: """The summary as text: one line per node, and the first error of a failing one.""" @@ -161,7 +172,9 @@ def __repr__(self) -> str: ) if node.failed: lines.append(f" first error: {_error_line(node.first_error)}") - lines.append(f" Full traceback: {_traceback_call(node.label)}") + lines.append( + f" Full traceback: {_traceback_call(node.label, self.target)}" + ) return "\n".join(lines) def _repr_html_(self) -> str: @@ -188,7 +201,7 @@ def _repr_html_(self) -> str: if node.failed: error = ( f"{html.escape(_error_line(node.first_error))}
" - f"{html.escape(_traceback_call(node.label))}" + f"{html.escape(_traceback_call(node.label, self.target))}" ) body.append( "" @@ -255,12 +268,41 @@ def _error_line(error: BaseException | None) -> str: return f"{type(error).__name__}: {first}" if first else type(error).__name__ -def _traceback_call(label: str) -> str: +def _traceback_call(label: str, target: str | None = None) -> str: """The call that prints a node's traceback, with the label as a valid string literal. - A JSON string is a valid Python string literal, and keeps double quotes. + A JSON string is a valid Python string literal, and keeps double quotes. A node + run's summary is kept on its node (``target``), a job run's on the job. + """ + owner = "job" if target is None else f"job.nodes[{json.dumps(target)}]" + return f"{owner}.last_run.print_traceback({json.dumps(label)})" + + +def warn_row_failures(summary: RunSummary, caller: FrameType) -> None: + """Issue the run's ``RowFailuresWarning``, shown after every run with failed rows. + + Python's default filter shows a warning once per text and line, so a second identical + failing run from the same line (re-running a notebook cell) would show nothing. A + fresh registry for each run keeps every run's warning, while the user's filters + (``ignore``, ``error``) still apply. + + Args: + summary: The run's summary, with failed rows. + caller: The frame that called ``run``; the warning is shown at its line. """ - return f"job.last_run.print_traceback({json.dumps(label)})" + import warnings + + from orcapod.errors import RowFailuresWarning + + warnings.warn_explicit( + summary.failures_message(), + RowFailuresWarning, + caller.f_code.co_filename, + caller.f_lineno, + module=caller.f_globals.get("__name__"), + registry=None, + module_globals=caller.f_globals, + ) class RunCountingObserver: @@ -351,7 +393,9 @@ def create_data_logger(self, tag: TagProtocol, data: DataProtocol) -> Any: return _NOOP_LOGGER -def run_node_summaries(dag: Any, counter: RunCountingObserver) -> list[NodeRunSummary]: +def run_node_summaries( + dag: Any, counter: RunCountingObserver, skip: Collection[Any] = () +) -> list[NodeRunSummary]: """One summary per function node of a finished run, in run order. The counts come from ``counter``; a load-only run's skipped inputs come from each @@ -361,6 +405,9 @@ def run_node_summaries(dag: Any, counter: RunCountingObserver) -> list[NodeRunSu Args: dag: The run's node graph (``topological_sort``, ``predecessors``). counter: The observer that counted the run's rows. + skip: Nodes of ``dag`` the run did not run (their rows were held): they get no + summary, but inputs they skipped in an earlier load-only run still make the + nodes below them incomplete. Returns: The summaries of the run's function nodes, in run order. @@ -383,6 +430,8 @@ def run_node_summaries(dag: Any, counter: RunCountingObserver) -> list[NodeRunSu if getattr(upstream, "last_run_not_stored", 0): above.add(id(upstream)) skipping_above[id(node)] = above + if any(node is skipped for skipped in skip): + continue if not (is_function_node(node) or is_side_effect_function_node(node)): continue labels = tuple( diff --git a/src/orcapod/pipeline/sync_orchestrator.py b/src/orcapod/pipeline/sync_orchestrator.py index af57145b3..a6515de5e 100644 --- a/src/orcapod/pipeline/sync_orchestrator.py +++ b/src/orcapod/pipeline/sync_orchestrator.py @@ -23,6 +23,8 @@ ) if TYPE_CHECKING: + from collections.abc import Mapping + from orcapod.pipeline.dag import GraphProtocol from orcapod.protocols.observability_protocols import ExecutionObserverProtocol from orcapod.protocols.core_protocols import DataProtocol, TagProtocol @@ -56,6 +58,7 @@ def run( run_id: str | None = None, pipeline_uri: str = "", load_only: bool = False, + held_rows: Mapping[Any, list[tuple[TagProtocol, DataProtocol]]] | None = None, ) -> OrchestratorResult: """Execute the node graph synchronously. @@ -74,6 +77,10 @@ def run( (ITL-1095): function nodes load their inputs' stored results and skip the others, operators run without storing, and a side-effect node passes its rows on without being called (nothing is delivered). + held_rows: Nodes of ``graph`` that are not run: each one's rows (the rows + of its current run, or of a stream outside the job) feed its + successors as if it had just run. Used by ``node.run()``, which runs + only the nodes it depends on that hold no current run (ITL-1095). Returns: OrchestratorResult with node outputs. @@ -91,9 +98,12 @@ def run( buffers: dict[Any, list[tuple[TagProtocol, DataProtocol]]] = {} processed: set[Any] = set() + held = held_rows if held_rows is not None else {} for node in topo_order: current = node - if is_source_node(node): + if node in held: + buffers[node] = list(held[node]) + elif is_source_node(node): buffers[node] = node.execute(observer=effective_observer) elif is_function_node(node) or is_side_effect_function_node(node): upstream_buf = self._gather_upstream(node, graph, buffers) diff --git a/tests/test_core/streams/test_run.py b/tests/test_core/streams/test_run.py index 54da7aecc..533e4785f 100644 --- a/tests/test_core/streams/test_run.py +++ b/tests/test_core/streams/test_run.py @@ -107,12 +107,21 @@ def _all_stream_classes() -> list[type]: @pytest.mark.parametrize("cls", _all_stream_classes(), ids=lambda c: c.__qualname__) -def test_every_stream_class_has_run_without_arguments(cls: type) -> None: - """``run`` exists on every stream class and takes no argument (developer decision).""" +def test_every_stream_class_has_run_without_positional_arguments(cls: type) -> None: + """``run`` exists on every stream class and takes no positional argument (developer + decision, ITL-1081); a job node's ``run`` takes one keyword-only argument, + ``load_only`` (ITL-1095).""" run = getattr(cls, "run", None) assert callable(run), f"{cls.__qualname__} has no run()" - parameters = list(inspect.signature(run).parameters) - assert parameters == ["self"], f"{cls.__qualname__}.run{parameters}" + parameters = inspect.signature(run).parameters + names = list(parameters) + if cls.__name__ in ("FunctionJobNode", "OperatorJobNode", "SourceJobNode"): + assert names == ["self", "load_only"], f"{cls.__qualname__}.run{names}" + load_only = parameters["load_only"] + assert load_only.kind is inspect.Parameter.KEYWORD_ONLY + assert load_only.default is False + else: + assert names == ["self"], f"{cls.__qualname__}.run{names}" def test_the_stream_protocol_declares_run() -> None: From 48a2015557ee74cf16102c0bb760537af47244b8 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 22:02:30 -0800 Subject: [PATCH 18/63] fix(pipeline): a run whose end-of-run commit fails leaves no node holding its rows (ITL-1095) job.run() (both orchestrators) and the node.run() cascade clear every function and operator job node of the run (not run, stale) before the commit's error propagates. A load-only run writes nothing, so its commit has nothing to fail on. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/pipeline/job.py | 4 ++ src/orcapod/pipeline/node_run.py | 22 ++++++ tests/test_pipeline/test_node_run_cascade.py | 73 ++++++++++++++++++++ 3 files changed, 99 insertions(+) diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 30cd42bbe..e36f6ca08 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -28,6 +28,7 @@ ) from orcapod.pipeline.base import AbstractPipelineBase from orcapod.pipeline.dag import OrcaDAG +from orcapod.pipeline.node_run import clear_run_rows from orcapod.pipeline.run_summary import ( RunCountingObserver, RunSummary, @@ -1237,10 +1238,13 @@ def _finish_run( except StoreReadError as exc: # A stored table could not be read: report the read failure itself. exc.add_note(pending_note) + # A run whose commit failed did not complete: no node holds its rows. + clear_run_rows(prepared.exec_dag.nodes()) raise except Exception as exc: flush_error = StoreWriteError(pipeline_db.base_path, flush=True) flush_error.add_note(pending_note) + clear_run_rows(prepared.exec_dag.nodes()) raise flush_error from exc # Mutate self in place. diff --git a/src/orcapod/pipeline/node_run.py b/src/orcapod/pipeline/node_run.py index 3cdf8d468..bfeae0e43 100644 --- a/src/orcapod/pipeline/node_run.py +++ b/src/orcapod/pipeline/node_run.py @@ -14,6 +14,7 @@ from __future__ import annotations +from collections.abc import Iterable from typing import TYPE_CHECKING, Any from orcapod.errors import StoreError, StoreWriteError @@ -66,6 +67,24 @@ def _job_name(node: Any) -> str: return "/".join(base_path) +def clear_run_rows(nodes: Iterable[Any]) -> None: + """Drop the rows of a run whose end-of-run commit failed from every node it filled. + + A run that raised defines no rows (ITL-1093): each function and operator job node of + the run is cleared (not run, stale) before the commit's error propagates. What was + committed stays stored. A load-only run writes nothing, so its commit has nothing + to fail on. + + Args: + nodes: The nodes of the run. + """ + from orcapod.core.nodes.staleness import RunHoldingNode + + for node in nodes: + if isinstance(node, RunHoldingNode): + node.clear_cache() # pyright: ignore[reportAttributeAccessIssue] # every RunHoldingNode is a job node with clear_cache + + def run_with_dependencies(target: Any, *, load_only: bool) -> RunSummary: """Run ``target`` and every job node it depends on that holds no current run. @@ -134,8 +153,11 @@ def visit(node: Any) -> None: try: store.flush() except StoreError: + # A run whose commit failed did not complete: no node holds its rows. + clear_run_rows(run_nodes) raise except Exception as exc: + clear_run_rows(run_nodes) raise StoreWriteError(store.base_path, flush=True) from exc return RunSummary( _job_name(target), diff --git a/tests/test_pipeline/test_node_run_cascade.py b/tests/test_pipeline/test_node_run_cascade.py index f986068ab..4d518933b 100644 --- a/tests/test_pipeline/test_node_run_cascade.py +++ b/tests/test_pipeline/test_node_run_cascade.py @@ -312,3 +312,76 @@ def test_the_cascade_commits_every_write(tmp_path: Path) -> None: assert CALLS == [] assert summary.nodes["rowwise"].reused == 6 assert summary.nodes["summed"].reused == 2 + + +class TestFailedFinalCommit: + """A run whose end-of-run commit fails leaves no node holding its rows.""" + + @staticmethod + def _flush_calls(run: Any) -> int: + """How many flushes ``run()`` makes on an in-memory store.""" + calls = 0 + original = InMemoryArrowDatabase.flush + + def counting(self: Any) -> None: + nonlocal calls + calls += 1 + original(self) + + with mock.patch.object(InMemoryArrowDatabase, "flush", counting): + run() + return calls + + @staticmethod + def _failing_last_flush(total_calls: int) -> Any: + """A flush that fails on call number ``total_calls`` (the final commit).""" + calls = 0 + original = InMemoryArrowDatabase.flush + + def flush(self: Any) -> None: + nonlocal calls + calls += 1 + if calls == total_calls: + raise OSError("disk full") + original(self) + + return mock.patch.object(InMemoryArrowDatabase, "flush", flush) + + @pytest.mark.parametrize("orchestrator", ["sync", "async"]) + def test_a_job_run_whose_commit_fails_leaves_every_node_empty( + self, orchestrator: str + ) -> None: + from orcapod.errors import StoreWriteError + from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator + + kinds = {"sync": SyncPipelineOrchestrator, "async": AsyncPipelineOrchestrator} + total = self._flush_calls( + lambda: _build(ROWS, InMemoryArrowDatabase()).run( + orchestrator=kinds[orchestrator]() + ) + ) + job = _build(ROWS, InMemoryArrowDatabase()) + with self._failing_last_flush(total), pytest.raises(StoreWriteError): + job.run(orchestrator=kinds[orchestrator]()) + for label in ("rowwise", "grouped", "summed"): + node = job.nodes[label] + assert node.as_table().num_rows == 0, label + assert node.head().status is ViewStatus.NOT_COMPUTED, label + assert node.is_stale, label + job.run(orchestrator=kinds[orchestrator]()) + assert _totals(job.nodes["summed"].as_table()) == [(0, 63), (1, 93)] + + def test_a_cascade_whose_commit_fails_leaves_every_node_empty(self) -> None: + from orcapod.errors import StoreWriteError + + total = self._flush_calls( + lambda: _build(ROWS, InMemoryArrowDatabase()).nodes["summed"].run() + ) + job = _build(ROWS, InMemoryArrowDatabase()) + with self._failing_last_flush(total), pytest.raises(StoreWriteError): + job.nodes["summed"].run() + for label in ("rowwise", "grouped", "summed"): + assert job.nodes[label].as_table().num_rows == 0, label + assert job.nodes[label].is_stale, label + view = job.nodes["summed"].run() + assert _totals(view.as_table()) == [(0, 63), (1, 93)] From 910842353ccc4d32feee1414fe9573609fb4bc47 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 22:03:26 -0800 Subject: [PATCH 19/63] docs: running one node, node states, and the run() argument rule (ITL-1095) Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 13 ++++++++++--- CLAUDE.md | 13 ++++++++++--- changelog.d/ITL-1095.md | 4 ++++ design_issues/ITL-1095-CR1.md | 8 ++++++-- docs/concepts/function-pods.md | 25 +++++++++++++++++++++++++ docs/concepts/streams.md | 2 +- 6 files changed, 56 insertions(+), 9 deletions(-) diff --git a/.zed/rules b/.zed/rules index f77ed3ea4..11173d2f4 100644 --- a/.zed/rules +++ b/.zed/rules @@ -498,10 +498,11 @@ conversion. Two specializations: Stream (core/streams/arrow_table_stream.py) — immutable (Tag, Data) sequence. Key methods: output_schema(), keys(), iter_data(), as_table(). -Every stream, source and node has run() with no arguments (developer, November 13, 2026, -ITL-1081): it does the work as_table() does (a job node runs as in a job) and returns a +Every stream, source and node has run() with no positional arguments (developer, November 13, +2026, ITL-1081): it does the work as_table() does (a job node runs as in a job) and returns a StreamView of every row; the view's head/tail/sort_by_tags choose what is shown. -head() and the notebook preview never compute. Never give run() arguments. +head() and the notebook preview never compute. A job node's run() takes one keyword-only +argument, load_only (ITL-1095); never give any other run() an argument. Source (core/sources/) — produces a stream from external data. ArrowTableSource is the core implementation; CSV/Delta/DataFrame/Dict/List sources all delegate to it internally. Each @@ -786,6 +787,12 @@ data. The rules: cannot be named (ITL-1043). job.run(load_only=True) computes nothing and writes nothing: function nodes load their own stored entries for this run's inputs and skip (and count) the others, operators run without storing, side-effect nodes pass rows on undelivered. + A job node's run(*, load_only=False) is state-driven: a node that holds a completed, + non-stale run (for a full run: not load-only, nor anything above it) runs nothing; + otherwise its non-current dependencies run first, each once, on the sync orchestrator + (pipeline/node_run.py), and the run commits as job.run() does. A failed end-of-run + commit clears every node of the run. A load-only run's rows have status + loaded_only. Only job nodes' run() takes an argument (load_only). - Empty data → ArrowTableSource raises ValueError("Table is empty"). - DerivedSource over a job node with nothing stored → presents zero rows with the origin's columns (no computed records; no error), in the plain form and with tags_as_data. diff --git a/CLAUDE.md b/CLAUDE.md index 93ff72c09..93de212f3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -513,10 +513,11 @@ conversion. Two specializations: **Stream** (`core/streams/arrow_table_stream.py`) — immutable (Tag, Data) sequence. Key methods: `output_schema()`, `keys()`, `iter_data()`, `as_table()`. -Every stream, source and node has `run()` with no arguments (developer, November 13, 2026, -ITL-1081): it does the work `as_table()` does (a job node runs as in a job) and returns a +Every stream, source and node has `run()` with no positional arguments (developer, November 13, +2026, ITL-1081): it does the work `as_table()` does (a job node runs as in a job) and returns a `StreamView` of every row; the view's `head`/`tail`/`sort_by_tags` choose what is shown. -`head()` and the notebook preview never compute. Never give `run()` arguments. +`head()` and the notebook preview never compute. A job node's `run()` takes one keyword-only +argument, `load_only` (ITL-1095); never give any other `run()` an argument. **Source** (`core/sources/`) — produces a stream from external data. `ArrowTableSource` is the core implementation; CSV/Delta/DataFrame/Dict/List sources all delegate to it internally. Each @@ -827,6 +828,12 @@ data. The rules: cannot be named (ITL-1043). `job.run(load_only=True)` computes nothing and writes nothing: function nodes load their own stored entries for this run's inputs and skip (and count) the others, operators run without storing, side-effect nodes pass rows on undelivered. + A job node's `run(*, load_only=False)` is state-driven: a node that holds a completed, + non-stale run (for a full run: not load-only, nor anything above it) runs nothing; + otherwise its non-current dependencies run first, each once, on the sync orchestrator + (`pipeline/node_run.py`), and the run commits as `job.run()` does. A failed end-of-run + commit clears every node of the run. A load-only run's rows have status + `loaded_only`. Only job nodes' `run()` takes an argument (`load_only`). - Empty data → `ArrowTableSource` raises `ValueError("Table is empty")`. - `DerivedSource` over a job node with nothing stored → presents zero rows with the origin's columns (no computed records; no error), in the plain form and with `tags_as_data`. diff --git a/changelog.d/ITL-1095.md b/changelog.d/ITL-1095.md index 8577bb28c..cc54a29fe 100644 --- a/changelog.d/ITL-1095.md +++ b/changelog.d/ITL-1095.md @@ -3,11 +3,15 @@ - A job node shows the current run; `node.as_source()` shows everything stored for it (developer, November 13, 2026; ITL-1095). Before a run in this process, a function or operator job node is empty: `as_table()`, `iter_data()`, the dataframe methods, `flow()`, `head()` and the notebook preview return zero rows with the node's columns, and `head()` (status `not_computed`) says the node has not run and points to `run()` and to `node.as_source()`. Before, a function node built over a store that already held results, or loaded with `PipelineJob.load`, returned every row stored under its key, and an operator node in `CacheMode.REPLAY` returned every logged run. The same holds after `clear_cache()`, after a run that raised, and while the node is stale. Migration: read stored results with `node.as_source()` (or `get_all_records()`), or run the job first; a run over a store that holds every result computes nothing. - A function or operator job node that has not run in this process is stale (`is_stale` is `True`; it has no modified time), and a job node is stale also when any function or operator job node it depends on is. Binding a source to a job's source slot (`job.bind(sources=...)`) marks the slot modified, so every node fed by it is stale and empty until it runs again. - A loaded operator node no longer raises from `as_table()` or `iter_data()`: it is empty, like every node that has not run. Its stored view (`as_source()`, `get_all_records()`) raises instead when the job file does not record its run's content hash and the store holds nothing under the hash the loaded job computes (ITL-1043). +- A job node's `run()` is state-driven and runs what the node depends on (developer, November 13, 2026; ITL-1095). A node that holds a completed run and is not stale runs nothing: `run()` returns the rows it holds. Otherwise every job node it depends on that holds no current run runs first, each once, as `job.run()` would run that part of the job, then the node; the run's writes are committed when it ends (before, `node.run()` read an upstream node's stored rows, or nothing, and left its last pipeline record and a `LOG` operator's output uncommitted). `run()` of a node whose function pod is a placeholder, or that was loaded without one, raises unless `load_only=True`; a `CACHE_ONLY` node's `run()` raises (its input is unavailable): read its stored results with `as_source()`. A job node's `run()` takes one keyword-only argument, `load_only`; every other stream's `run()` still takes none. +- If the commit at the end of `job.run()` or `node.run()` fails, every function and operator node of the run is cleared (not run, stale) before the error propagates. ### Added - `job.run(load_only=True)` and `await job.run_async(load_only=True)` fill a job's nodes with what is already stored for this run's inputs and compute nothing (ITL-1095): sources are read, operators run on the rows that reach them (a `CacheMode.LOG` cache writes nothing), each function node takes the stored results of its inputs and skips the inputs without one, a side-effect node passes its rows on without delivering anything, and nothing is written to the store. `RunSummary.load_only`, `NodeRunSummary.not_stored` (inputs skipped) and `NodeRunSummary.inputs_incomplete` (the function nodes above a node that skipped inputs) report it; loaded rows count in `reused`, and the summary's text says "3 loaded, 1 not stored (skipped)". +- `node.run(load_only=True)`, and `node.last_run`, the `RunSummary` of a node's last `run()`. `ViewStatus.LOADED_ONLY`: the `head()` and `run()` view of a node filled by a load-only run, captioned "N loaded, M inputs without a stored result" and naming the function nodes above it that skipped inputs. `holds_current_run(load_only=...)` on function and operator job nodes says whether `run()` would run them. + ### Fixed - On the asynchronous orchestrator, an operator node in `CacheMode.REPLAY` stamps itself after its inputs close, so the rows it replayed are not judged out of date by the time the run ends. diff --git a/design_issues/ITL-1095-CR1.md b/design_issues/ITL-1095-CR1.md index b2e853b2c..ff78f3191 100644 --- a/design_issues/ITL-1095-CR1.md +++ b/design_issues/ITL-1095-CR1.md @@ -20,5 +20,9 @@ shows everything stored. Function and operator job nodes read empty (typed zero `clear_cache()`, after a run that raised and while stale; no read touches the store. A never-run or cleared node has no modified time (stale); `core/nodes/staleness.py` makes a job node stale when any function or operator job node above it is; a `SourceJobNode` is modified when bound or -when its bound source is. Planned in the same issue: `run(load_only=True)` and the `node.run()` -cascade. +when its bound source is. `job.run(load_only=True)` fills the nodes from what is stored and +computes and writes nothing. A job node's `run(*, load_only=False)` is state-driven: a current +node runs nothing; otherwise its non-current dependencies run first, once each, on the sync +orchestrator restricted to them (`pipeline/node_run.py`), and the run commits as `job.run()` +does; a failed end-of-run commit clears every node of the run. A load-only run's rows have +status `loaded_only` and are never current for a full run. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index cbb4a4415..2492f6349 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -578,6 +578,31 @@ incomplete: a group formed after it may lack members, and the result stored for group by an earlier run may then be loaded for it. `run_async(load_only=True)` does the same on the asynchronous orchestrator. +### Running one node: `node.run()` + +`job.nodes[label].run()` runs that node as `job.run()` would, together with the job nodes +it depends on, and returns a view of its rows. What runs depends on each node's state +(developer, November 13, 2026): + +| State of a node | What it shows | `run()` | `run(load_only=True)` | +|---|---|---|---| +| Not run in this process (a fresh or loaded job), cleared, or stale | nothing: `head()` says "not run" (`not_computed`) | runs it | loads it | +| Filled by a load-only run | the rows it loaded: status `loaded_only`, "N loaded, M inputs without a stored result" | runs it, computing what it skipped | nothing: shows what it holds | +| Filled by a run (`job.run()` or `run()`) | that run's rows (`rows`) | nothing: shows what it holds | nothing: shows what it holds | + +A node is stale when anything it depends on changed after its run: an upstream node ran +again or was cleared, or a source was bound again (`job.bind(sources=...)`). Before a node +runs, every job node it depends on that is not current for that kind of run runs first, +each once, in order; the others feed it their rows. A load-only run's rows are never current +for a full run, nor are the rows of any node below them. `node.last_run` holds the counts, +as `job.last_run` does for `job.run()`, and the run's writes are committed when it ends. +Side effects above the node are delivered as `job.run()` delivers them; a node that holds a +current run is not run again, so to deliver a side effect again (with +`track_completion=False`), call `clear_cache()` on its node first. Staleness comes from +modified times only: a source whose rows change without its object changing (a table read +lazily, a file rewritten in place) is not seen. Build a new source object and bind it, or +clear the nodes, to run them again. + ### When results are committed Inside `job.run()`, each function node commits its results and pipeline records to the diff --git a/docs/concepts/streams.md b/docs/concepts/streams.md index 838cc31fe..7d61ad3ba 100644 --- a/docs/concepts/streams.md +++ b/docs/concepts/streams.md @@ -205,7 +205,7 @@ What `run()` does on each kind of object: | A `PollingSource` | polls, as `as_table()` does | | A `CachedSource` | reads its cache and writes the wrapped source's new rows to it | | A function pod's or operator's output stream | computes what is not computed yet | -| A job node | runs the node as a job run would (`as_table()` of a job node only reads), then returns the view of that run's rows | +| A job node | runs the node as a job run would, after every job node it depends on that holds no current run (each once), commits, and returns the view of that run's rows; a node that holds a current run (completed in this process and not stale) runs nothing. `run(load_only=True)` loads stored results only. `as_table()` of a job node only reads | | A side-effect stream | delivers its side effects for every row, on every call, and returns the rows it passed on; `head()` still shows none | | A pipeline blueprint node, an unbound input slot | raises, as `as_table()` does | From a9d07f06a5eae40e48df03adbe1ddf871475152a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 22:08:07 -0800 Subject: [PATCH 20/63] fix(function_node): any raise of a run clears the node's rows, also a refusal or an observer failure (ITL-1093) Addresses the second CodeRabbit review of PR #378: a sync run refused for its input's data context, or failing in on_node_end, kept the previous run's rows. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/function_node.py | 28 ++++++++----- .../test_job_node_rows_after_run.py | 40 +++++++++++++++++++ .../test_pipeline/test_stored_data_context.py | 32 +++++++++++++++ 3 files changed, 91 insertions(+), 9 deletions(-) diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 252c98c6e..1f81a2f34 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -1940,11 +1940,17 @@ def execute( if self._function_pod is None: raise self._cannot_compute_error() - self._begin_run() - # A whole-input condition, not a row failure: the error policy does not apply. - require_same_data_context( - input_stream.data_context_key, self._stored_data_context_key - ) + try: + self._begin_run() + # A whole-input condition, not a row failure: the error policy does not + # apply. + require_same_data_context( + input_stream.data_context_key, self._stored_data_context_key + ) + except BaseException: + # A run that raised defines no rows, also when it was refused at once. + self.clear_cache() + raise node_label = self.label node_hash = self.content_hash().to_string() @@ -2041,6 +2047,9 @@ def execute( self._commit_or_end_node( self._commit_at_node_end, ctx_obs, node_label, node_hash ) + # The node's rows are those of this run's inputs, whatever else it caches. + self._set_run_rows(emitted_entry_ids) + ctx_obs.on_node_end(node_label, node_hash) except Exception as run_error: self._commit_after_failure(run_error) # A run that raised defines no rows: the node reads as one with no run. @@ -2050,9 +2059,6 @@ def execute( self.clear_cache() raise - # The node's rows are those of this run's inputs, whatever else it caches. - self._set_run_rows(emitted_entry_ids) - ctx_obs.on_node_end(node_label, node_hash) # Mark this node as freshly computed so subsequent iter_data() calls # skip the is_stale check and serve results directly from the in-memory cache. self._update_modified_time() @@ -3389,7 +3395,9 @@ async def async_execute( try: self._begin_run() except StoreError: - # Nothing was delivered; downstream must not wait for rows. + # Nothing was delivered; downstream must not wait for rows. A run that + # raised defines no rows. + self.clear_cache() await output.close() raise status = self.load_status @@ -3414,6 +3422,8 @@ async def async_execute( self._input_stream.data_context_key, self._stored_data_context_key ) except InputValidationError: + # A run that raised defines no rows, also when it was refused at once. + self.clear_cache() await output.close() raise node_label = self.label diff --git a/tests/test_pipeline/test_job_node_rows_after_run.py b/tests/test_pipeline/test_job_node_rows_after_run.py index e1a53ee7e..e45ff97bc 100644 --- a/tests/test_pipeline/test_job_node_rows_after_run.py +++ b/tests/test_pipeline/test_job_node_rows_after_run.py @@ -435,3 +435,43 @@ def test_async_one_of_two_equal_inputs_fails(self) -> None: received = self._run_async(node) assert received == [(0, 6), (2, 8)] assert self._held(node) == received + + +class TestRefusedOrFailedRunAfterAGoodRun: + """A run that raises after a good run drops that run's rows too (ITL-1093). + + The node then reads as a node with no run in this process: every row stored under + its content hash, read from the store, whatever raised. An input refused for its data + context is tested in ``test_stored_data_context.py``, which registers a second one. + """ + + @staticmethod + def _job(db: Any) -> PipelineJob: + """Return a job applying ``inc`` to a source of 6 rows as ``node``.""" + return _single(inc.pod, ROWS, db) + + @pytest.mark.parametrize("kind", ORCHESTRATORS) + def test_observer_failure_at_node_end_drops_the_rows(self, kind: str) -> None: + """An observer raising at the node's end leaves the node with no run's rows.""" + from orcapod.pipeline.observer import NoOpObserver + + class FailsAtNodeEnd(NoOpObserver): + """An observer whose ``on_node_end`` raises.""" + + def on_node_end(self, node_label: str, node_hash: str) -> None: + """Raise at the function node's end, as a broken observer would.""" + if node_label == "node": + raise RuntimeError("observer failed") + + db = InMemoryArrowDatabase() + job = self._job(db) + job.run(orchestrator=_orchestrator(kind)) + job.bind(sources={"s": _source(ROWS[:3])}) + job.run(orchestrator=_orchestrator(kind)) + node = job.nodes["node"] + assert _values(node, lambda r: r["k"]) == [0, 1, 2] + # The async orchestrator raises its task group's error around it. + with pytest.raises((RuntimeError, ExceptionGroup)): + job.run(orchestrator=_orchestrator(kind), observer=FailsAtNodeEnd()) + # Every stored row (the 6-row run's), read from the store. + assert _values(node, lambda r: r["k"]) == [0, 1, 2, 3, 4, 5] diff --git a/tests/test_pipeline/test_stored_data_context.py b/tests/test_pipeline/test_stored_data_context.py index 49708925a..17f072b73 100644 --- a/tests/test_pipeline/test_stored_data_context.py +++ b/tests/test_pipeline/test_stored_data_context.py @@ -1254,6 +1254,38 @@ def test_function_node_raises_and_writes_nothing(self, store: Store): assert calls == [] assert reopened(store).list_sources() == [] + def test_refused_input_after_a_good_run_leaves_no_run_rows(self, store: Store): + """A run refused for its input's data context defines no rows (ITL-1093). + + After a good run the node holds that run's rows; a later sync run refused for its + input's data context drops them, and the node reads its stored rows, as with no + run in this process. (On the async path the context checked is the node's own + input stream's, so a node refused there never had a run.) + """ + table = pa.table( + {"k": ["a", "b"], "x": [1, 2]}, + schema=pa.schema( + [ + pa.field("k", pa.large_string(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] + ), + ) + node = FunctionJobNode( + function_pod=double.pod, # pyright: ignore[reportArgumentType] # the decorator types .pod as the pod base class + input_stream=ArrowTableStream(table, tag_columns=["k"]), + pipeline_database=store, + result_database=store, + ) + node.run() + assert node.head(10).origin == "held" + + with pytest.raises(InputValidationError): + node.execute(_other_context_stream()) + + assert node.head(10).origin == "stored" + assert sorted(row["double"] for row in node.as_table().to_pylist()) == [2, 4] + def test_add_pipeline_record_refuses_an_input_of_another_data_context( self, store: Store ): From 2930456789de16251ee464d4f3b6cdb2b0c4dc58 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 22:10:18 -0800 Subject: [PATCH 21/63] test(function_node): a CACHE_ONLY node's run() raises and points to as_source() (ITL-1095) Co-Authored-By: Claude Opus 5.5 --- tests/test_core/nodes/test_function_node_iteration.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_core/nodes/test_function_node_iteration.py b/tests/test_core/nodes/test_function_node_iteration.py index e28d54b4d..10a5266a9 100644 --- a/tests/test_core/nodes/test_function_node_iteration.py +++ b/tests/test_core/nodes/test_function_node_iteration.py @@ -123,10 +123,10 @@ def test_as_table_fresh_node_returns_empty_no_compute(self): assert "id" in table.column_names assert "result" in table.column_names - def test_run_cache_only_is_noop(self): - """run() on a CACHE_ONLY node computes nothing and returns the stored rows' view.""" + def test_run_cache_only_raises_and_points_to_the_stored_view(self): + """run() on a CACHE_ONLY node computes nothing: its input is unavailable, so it + has nothing to run or look up, and it says to read ``as_source()`` (ITL-1095).""" from orcapod.pipeline.serialization import LoadStatus - from orcapod.streams import StreamView node = _make_node() # A loaded node whose upstream is unavailable keeps that input stream; only its @@ -134,9 +134,10 @@ def test_run_cache_only_is_noop(self): node._load_status = LoadStatus.CACHE_ONLY with patch.object(node, "execute") as mock_exec: - view = node.run() + for load_only in (False, True): + with pytest.raises(RuntimeError, match=r"as_source\(\)"): + node.run(load_only=load_only) mock_exec.assert_not_called() - assert isinstance(view, StreamView) def test_run_unavailable_raises(self): """run() on an UNAVAILABLE node raises RuntimeError.""" From b693b50e336ee66e1b193649c0500c593ece43c3 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 22:11:12 -0800 Subject: [PATCH 22/63] test: #378's new raised-run tests read the stored view (ITL-1095); load-only path clears on a refused run Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/function_node.py | 14 +++++++------- .../test_pipeline/test_job_node_rows_after_run.py | 6 ++++-- tests/test_pipeline/test_stored_data_context.py | 14 +++++++++----- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index d48ff8b42..2d4513dab 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -1627,21 +1627,21 @@ def _execute_load_only( from orcapod.pipeline.observer import NoOpObserver from orcapod.pipeline.serialization import LoadStatus - if self.load_status == LoadStatus.UNAVAILABLE: - raise self._no_store_error() - require_same_data_context( - input_stream.data_context_key, self._stored_data_context_key - ) node_label = self.label node_hash = self.content_hash().to_string() obs = observer if observer is not None else NoOpObserver() ctx_obs = obs.contextualize(*self.node_identity_path) - tag_schema = input_stream.output_schema(columns={"system_tags": True})[0] - ctx_obs.on_node_start(node_label, node_hash, tag_schema=tag_schema) output: list[tuple[TagProtocol, DataProtocol]] = [] loaded_ids: list[bytes] = [] not_stored = 0 try: + if self.load_status == LoadStatus.UNAVAILABLE: + raise self._no_store_error() + require_same_data_context( + input_stream.data_context_key, self._stored_data_context_key + ) + tag_schema = input_stream.output_schema(columns={"system_tags": True})[0] + ctx_obs.on_node_start(node_label, node_hash, tag_schema=tag_schema) inputs = [ (tag, data, self.compute_base_entry_id(tag, data)) for tag, data in input_stream.iter_data() diff --git a/tests/test_pipeline/test_job_node_rows_after_run.py b/tests/test_pipeline/test_job_node_rows_after_run.py index b98c66655..6634a4fdb 100644 --- a/tests/test_pipeline/test_job_node_rows_after_run.py +++ b/tests/test_pipeline/test_job_node_rows_after_run.py @@ -467,5 +467,7 @@ def on_node_end(self, node_label: str, node_hash: str) -> None: # The async orchestrator raises its task group's error around it. with pytest.raises((RuntimeError, ExceptionGroup)): job.run(orchestrator=_orchestrator(kind), observer=FailsAtNodeEnd()) - # Every stored row (the 6-row run's), read from the store. - assert _values(node, lambda r: r["k"]) == [0, 1, 2, 3, 4, 5] + # The node holds no run (it reads empty, ITL-1095); the store keeps every row. + assert _values(node, lambda r: r["k"]) == [] + stored = node.as_source().as_table().to_pylist() + assert sorted(r["k"] for r in stored) == [0, 1, 2, 3, 4, 5] diff --git a/tests/test_pipeline/test_stored_data_context.py b/tests/test_pipeline/test_stored_data_context.py index 79b2356ab..a3d678727 100644 --- a/tests/test_pipeline/test_stored_data_context.py +++ b/tests/test_pipeline/test_stored_data_context.py @@ -48,6 +48,7 @@ from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator from orcapod.side_effects import SideEffectPod, sink_pod from orcapod.sources import ArrowTableSource, DictSource +from orcapod.core.streams import ViewStatus from orcapod.types import CacheMode, NodeConfig Orchestrator = SyncPipelineOrchestrator | AsyncPipelineOrchestrator @@ -1266,9 +1267,10 @@ def test_refused_input_after_a_good_run_leaves_no_run_rows(self, store: Store): """A run refused for its input's data context defines no rows (ITL-1093). After a good run the node holds that run's rows; a later sync run refused for its - input's data context drops them, and the node reads its stored rows, as with no - run in this process. (On the async path the context checked is the node's own - input stream's, so a node refused there never had a run.) + input's data context drops them, and the node reads as with no run in this + process: empty (ITL-1095); its stored view keeps the rows. (On the async path the + context checked is the node's own input stream's, so a node refused there never + had a run.) """ table = pa.table( {"k": ["a", "b"], "x": [1, 2]}, @@ -1291,8 +1293,10 @@ def test_refused_input_after_a_good_run_leaves_no_run_rows(self, store: Store): with pytest.raises(InputValidationError): node.execute(_other_context_stream()) - assert node.head(10).origin == "stored" - assert sorted(row["double"] for row in node.as_table().to_pylist()) == [2, 4] + assert node.head(10).status is ViewStatus.NOT_COMPUTED + assert node.as_table().num_rows == 0 + stored = node.as_source().as_table().to_pylist() + assert sorted(row["double"] for row in stored) == [2, 4] def test_add_pipeline_record_refuses_an_input_of_another_data_context( self, store: Store From 3610fd20d59c05a0e78f4554f3d378999eb71231 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 22:13:47 -0800 Subject: [PATCH 23/63] feat(pipeline): a loaded job is filled from its store by run(load_only=True) (ITL-1095) loaded.run(load_only=True) and a loaded node's run(load_only=True) read the job's sources (rebuilt or bound), run the rebuilt operators (a loaded REPLAY operator replays) and give each function node the results its store holds for those inputs; nodes below an unbound source stay empty. loaded.run() still raises, now naming run(load_only=True) and as_source(). The four loaded-job tests that waited for this stage read through a load-only run again (no xfails left). Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 1 + CLAUDE.md | 1 + changelog.d/ITL-1095.md | 2 + docs/concepts/function-pods.md | 21 ++- src/orcapod/core/nodes/function_node.py | 4 +- src/orcapod/core/nodes/operator_node.py | 10 +- src/orcapod/pipeline/job.py | 32 ++-- .../loaded_job/test_loaded_job_readback.py | 101 ++++++----- .../loaded_job/test_loaded_load_only.py | 166 ++++++++++++++++++ 9 files changed, 276 insertions(+), 62 deletions(-) create mode 100644 tests/test_pipeline/loaded_job/test_loaded_load_only.py diff --git a/.zed/rules b/.zed/rules index 11173d2f4..94beea7a3 100644 --- a/.zed/rules +++ b/.zed/rules @@ -793,6 +793,7 @@ data. The rules: (pipeline/node_run.py), and the run commits as job.run() does. A failed end-of-run commit clears every node of the run. A load-only run's rows have status loaded_only. Only job nodes' run() takes an argument (load_only). + A loaded job (PipelineJob.load) runs only with load_only=True. - Empty data → ArrowTableSource raises ValueError("Table is empty"). - DerivedSource over a job node with nothing stored → presents zero rows with the origin's columns (no computed records; no error), in the plain form and with tags_as_data. diff --git a/CLAUDE.md b/CLAUDE.md index 93de212f3..4025641ff 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -834,6 +834,7 @@ data. The rules: (`pipeline/node_run.py`), and the run commits as `job.run()` does. A failed end-of-run commit clears every node of the run. A load-only run's rows have status `loaded_only`. Only job nodes' `run()` takes an argument (`load_only`). + A loaded job (`PipelineJob.load`) runs only with `load_only=True`. - Empty data → `ArrowTableSource` raises `ValueError("Table is empty")`. - `DerivedSource` over a job node with nothing stored → presents zero rows with the origin's columns (no computed records; no error), in the plain form and with `tags_as_data`. diff --git a/changelog.d/ITL-1095.md b/changelog.d/ITL-1095.md index cc54a29fe..543769b5f 100644 --- a/changelog.d/ITL-1095.md +++ b/changelog.d/ITL-1095.md @@ -12,6 +12,8 @@ - `node.run(load_only=True)`, and `node.last_run`, the `RunSummary` of a node's last `run()`. `ViewStatus.LOADED_ONLY`: the `head()` and `run()` view of a node filled by a load-only run, captioned "N loaded, M inputs without a stored result" and naming the function nodes above it that skipped inputs. `holds_current_run(load_only=...)` on function and operator job nodes says whether `run()` would run them. +- A job loaded with `PipelineJob.load` can be filled from its store: `loaded.run(load_only=True)` (and `node.run(load_only=True)` of a loaded node) reads the job's sources (rebuilt from files, or bound with `loaded.bind(sources=...)`), runs the rebuilt operators and gives each function node the results its store holds for those inputs; nodes below an unbound source stay empty. `loaded.run()` without `load_only` still raises, and says how to load. + ### Fixed - On the asynchronous orchestrator, an operator node in `CacheMode.REPLAY` stamps itself after its inputs close, so the rows it replayed are not judged out of date by the time the run ends. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 2492f6349..785369fd0 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -765,10 +765,23 @@ print(loaded.nodes["bmi"].as_source().as_table().sort_by("subject_id").to_pandas # 2 mouse_03 2297.520661 ``` -A loaded job is for reading stored results, through each node's stored view, -`node.as_source()`: its nodes have not run in this process, so they are empty. It cannot be -run again: function pods are not saved with the job, so `loaded.run()` raises. To compute -again, build the job from your code as before. +A loaded job is for reading stored results. Its nodes have not run in this process, so +they are empty; each node's stored view, `node.as_source()`, reads what the store holds. A +loaded job cannot compute: function pods are not saved with it, so `loaded.run()` raises. +`loaded.run(load_only=True)` fills its nodes from the store instead, as for any job (see +*Loading without computing* above): the sources are read (an in-memory source must be bound +first, since its data is not saved), the rebuilt operators run, and each function node takes +the results stored for those inputs. Nodes below a source that is not available stay empty. +`node.run(load_only=True)` does the same for one node and what it depends on. + +```python +loaded.bind(sources={"mice": DictSource(data=mice, tag_columns=["subject_id"], source_id="mice")}) +summary = loaded.run(load_only=True) +print(loaded.nodes["bmi"].as_table().num_rows, summary.nodes["bmi"].not_stored) +# 3 0 +``` + +To compute again, build the job from your code as before. The store is the source of truth: a loaded job's stored views show what the store holds for it. The job file names each run's rows by the content hash the run stored them under, so a diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 2d4513dab..39aca8292 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -1778,8 +1778,8 @@ def _cannot_compute_error(self) -> RuntimeError: """ return RuntimeError( f"FunctionJobNode {self.label!r} was loaded by PipelineJob.load and has no " - "function pod, so it cannot compute; read its stored results with " - "as_source()." + "function pod, so it cannot compute; fill it from the store with " + "run(load_only=True), or read its stored results with as_source()." ) def _stored_record_count(self) -> int: diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index a9aa8ec30..9725f928b 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -1018,7 +1018,9 @@ def _cannot_compute_error(self) -> RuntimeError: """Describe why this loaded node cannot execute an operator.""" return RuntimeError( f"OperatorJobNode {self.label!r} was loaded by PipelineJob.load and " - "cannot compute; create a fresh PipelineJob to run the pipeline again." + "cannot compute; fill it from the store with run(load_only=True), read " + "its stored results with as_source(), or create a fresh PipelineJob to " + "run the pipeline again." ) def _filter_by_content_hash(self, table: pa.Table) -> pa.Table: @@ -1341,7 +1343,8 @@ def execute( """ from orcapod.pipeline.observer import NoOpObserver - if self._loaded: + # A loaded node runs only in a load-only run, with its rebuilt operator. + if self._loaded and (not load_only or self._operator is None): raise self._cannot_compute_error() node_label = self.label @@ -1753,7 +1756,8 @@ async def async_execute( """ from orcapod.pipeline.observer import NoOpObserver - if self._loaded: + # A loaded node runs only in a load-only run, with its rebuilt operator. + if self._loaded and (not load_only or self._operator is None): await output.close() raise self._cannot_compute_error() diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index e36f6ca08..91644b9a5 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -929,7 +929,7 @@ def run( """ from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator - prepared = self._prepare_run(commit_every_rows) + prepared = self._prepare_run(commit_every_rows, load_only=load_only) counter = RunCountingObserver() effective_orchestrator = orchestrator or SyncPipelineOrchestrator() with self._executing(prepared, commit_every_rows): @@ -992,7 +992,7 @@ async def run_async( f"{type(orchestrator).__name__}. Use job.run(orchestrator=...) to run " "it." ) - prepared = self._prepare_run(commit_every_rows) + prepared = self._prepare_run(commit_every_rows, load_only=load_only) counter = RunCountingObserver() with self._executing(prepared, commit_every_rows): _ = await orchestrator.run_async( @@ -1008,11 +1008,14 @@ async def run_async( warn_row_failures(summary, sys._getframe(1)) # pyright: ignore[reportPrivateUsage] # the caller's line is where the warning is shown return summary - def _prepare_run(self, commit_every_rows: int) -> "_PreparedRun": + def _prepare_run( + self, commit_every_rows: int, *, load_only: bool = False + ) -> "_PreparedRun": """Check the job can run and build what a run needs; shared by both run forms. Args: commit_every_rows: As for ``run``. + load_only: As for ``run``: a loaded job can run only this way. Returns: The execution DAG, run id, pipeline URI and the store views of the run. @@ -1039,14 +1042,17 @@ def _prepare_run(self, commit_every_rows: int) -> "_PreparedRun": "PipelineJob has no compiled pipeline. " "Record invocations inside a 'with job:' block first." ) - # A loaded job reads the results its saved run stored and never runs: - # function pods are not saved with it, and even an operator-only loaded - # job, whose operators are rebuilt, stays read-only. - if self._loaded: + # A loaded job never computes: function pods are not saved with it, and even + # an operator-only loaded job, whose operators are rebuilt, stays read-only. + # A load-only run fills it from the store (ITL-1095). + if self._loaded and not load_only: raise RuntimeError( - "PipelineJob.run() cannot execute a loaded job: PipelineJob.load() " - "is for reading stored results only. To run a new computation, " - "create a fresh PipelineJob from the original pipeline definition." + "PipelineJob.run() cannot execute a loaded job: its function pods are " + "not saved with it, so it cannot compute. Fill its nodes from the " + "store with run(load_only=True) (bind in-memory sources first), or " + "read a node's stored results with node.as_source(). To run a new " + "computation, create a fresh PipelineJob from the original pipeline " + "definition." ) # Until this run finishes, what the store holds for a node is not certain. self._last_run_finished = False @@ -1087,7 +1093,11 @@ def _prepare_run(self, commit_every_rows: int) -> "_PreparedRun": isinstance(node, (FunctionJobNode, OperatorJobNode)) and node.load_status == LoadStatus.UNAVAILABLE ) - or (isinstance(node, FunctionJobNode) and node.producer is None) + or ( + not load_only + and isinstance(node, FunctionJobNode) + and node.producer is None + ) ] if unavailable_labels: raise RuntimeError( diff --git a/tests/test_pipeline/loaded_job/test_loaded_job_readback.py b/tests/test_pipeline/loaded_job/test_loaded_job_readback.py index b14dff36e..53bd101b5 100644 --- a/tests/test_pipeline/loaded_job/test_loaded_job_readback.py +++ b/tests/test_pipeline/loaded_job/test_loaded_job_readback.py @@ -69,6 +69,39 @@ def _rewrite_job_file( return str(path) +def _loaded_and_filled(saved: pathlib.Path) -> PipelineJob: + """Load the saved job, bind its in-memory sources and fill it from the store. + + A loaded node is empty until a load-only run fills it (ITL-1095). The sources bound + are those of ``save_job_script.py``; the CSV and Delta sources are rebuilt. + """ + from orcapod.sources import ArrowTableSource, ListSource + + loaded = _load(saved) + arrow = ArrowTableSource( + pa.table( + {"id": [0, 1], "y": [3, 4]}, + schema=pa.schema( + [ + pa.field("id", pa.int64(), nullable=False), + pa.field("y", pa.int64(), nullable=False), + ] + ), + ), + tag_columns=["id"], + source_id="asrc", + ) + loaded.bind( + sources={ + "dsrc": _dict_source(), + "asrc": arrow, + "lsrc": ListSource("z", [1, 2], source_id="lsrc"), + } + ) + _ = loaded.run(load_only=True) + return loaded + + def _dict_source() -> DictSource: """Return a two-row in-memory source with tag ``id`` and data ``x``.""" return DictSource( @@ -235,22 +268,18 @@ def test_loaded_function_node_as_table_returns_stored_rows( assert _sorted(table, TAG_OF[label]).equals(_sorted(expected, TAG_OF[label])) -@pytest.mark.xfail( - strict=True, - reason="ITL-1095: a loaded node reads empty until loaded.run(load_only=True), " - "which reads its entries this way (stage 4)", -) @pytest.mark.parametrize("label", FUNCTION_NODES) def test_loaded_function_node_as_table_all_info_has_stream_columns( saved: pathlib.Path, label: str ) -> None: - """With ``all_info``, a loaded node's table has every live column and more.""" + """With ``all_info``, a loaded node's table, once a load-only run has filled it, + has every live column and more.""" # The loaded table follows the documented stream semantics: every column of the # live table, with the same values and types, plus the data context and the # source-info columns. (The live FunctionJobNode.as_table() drops those two # groups -- a separate, pre-existing bug the loaded node does not copy.) expected = pq.read_table(saved / "expected" / f"{label}.all_info.parquet") - table = _load(saved).nodes[label].as_source().as_table(all_info=True) + table = _loaded_and_filled(saved).nodes[label].as_table(all_info=True) data_column = next(iter(_load(saved).nodes[label].output_schema()[1])) assert "_context_key" in table.column_names assert f"_source_{data_column}" in table.column_names @@ -388,15 +417,11 @@ def test_other_context_records_do_not_satisfy_saved_expectation( assert rows is None or len(rows) == 0 -@pytest.mark.xfail( - strict=True, - reason="ITL-1095: a loaded node reads empty until loaded.run(load_only=True), " - "which reads its entries this way (stage 4)", -) def test_recomputed_entry_is_read_once_with_its_latest_result( tmp_path: pathlib.Path, ) -> None: - """A loaded node reads one row per entry: the greatest recomputation index wins.""" + """A loaded node's load-only run reads one row per entry: the greatest recomputation + index wins.""" path = _save_small_job(tmp_path, _record_double) node = PipelineJob.load(path).nodes["dbl"] # A loaded node is empty until it runs (ITL-1095): read its stored view. @@ -405,7 +430,10 @@ def test_recomputed_entry_is_read_once_with_its_latest_result( _append_recomputed_record(node, entry_id, recomputation_index=2, value=222) _append_recomputed_record(node, entry_id, recomputation_index=1, value=111) - reloaded = PipelineJob.load(path).nodes["dbl"].as_source() + reloaded_job = PipelineJob.load(path) + reloaded_job.bind(sources={"dsrc": _dict_source()}) + _ = reloaded_job.run(load_only=True) + reloaded = reloaded_job.nodes["dbl"] table = reloaded.as_table() assert table.num_rows == 2 rows = {row["id"]: row["dbl"] for row in table.to_pylist()} @@ -442,15 +470,11 @@ def test_live_node_hot_load_reads_the_latest_recomputed_result( assert values[entry_id] == 222 -@pytest.mark.xfail( - strict=True, - reason="ITL-1095: a loaded node reads empty until loaded.run(load_only=True), " - "which reads its entries this way (stage 4)", -) def test_loaded_node_with_ephemeral_results_does_not_crash( tmp_path: pathlib.Path, ) -> None: - """Ephemeral results come back as placeholders; the table says why it has none.""" + """Results kept in an ephemeral store are gone after the run: a load-only run of the + loaded job counts them as not stored and shows no rows; it does not crash.""" @function_pod(output_keys="dbl") def double(x: int) -> int: @@ -468,25 +492,14 @@ def make_ephemeral(job: PipelineJob) -> None: node.set_ephemeral_store(InMemoryArrowDatabase()) path = _save_small_job(tmp_path, record, configure=make_ephemeral) - node = PipelineJob.load(path).nodes["dbl"] - assert [type(data).__name__ for _, data in node.iter_data()] == [ - "EmptyData", - "EmptyData", - ] - with pytest.raises(RuntimeError) as excinfo: - _ = node.as_table() - assert str(excinfo.value) == ( - "FunctionJobNode 'dbl': the saved run kept 2 of its results in an ephemeral " - "store, which is not saved with the job, so they cannot be read into a table. " - "iter_data() yields them as EmptyData placeholders." - ) + loaded = PipelineJob.load(path) + loaded.bind(sources={"dsrc": _dict_source()}) + summary = loaded.run(load_only=True) + assert summary.nodes["dbl"].not_stored == 2 + assert summary.nodes["dbl"].reused == 0 + assert loaded.nodes["dbl"].as_table().num_rows == 0 -@pytest.mark.xfail( - strict=True, - reason="ITL-1095: a loaded node reads empty until loaded.run(load_only=True), " - "which reads its entries this way (stage 4)", -) def test_newer_unavailable_result_does_not_expose_old_persistent_result( tmp_path: pathlib.Path, ) -> None: @@ -509,10 +522,13 @@ def test_newer_unavailable_result_does_not_expose_old_persistent_result( records = records.set_column(index, field, pa.array(values, type=field.type)) store.add_records(table_path, records, record_id_column="__rid", flush=True) - node = PipelineJob.load(path).nodes["dbl"] - assert [type(data).__name__ for _, data in node.iter_data()] == ["EmptyData"] * 2 - with pytest.raises(RuntimeError, match="ephemeral store"): - node.as_table() + # The newest result of each entry is gone with its ephemeral store: a load-only run + # counts the entry as not stored and never shows the older persistent result. + loaded = PipelineJob.load(path) + loaded.bind(sources={"dsrc": _dict_source()}) + summary = loaded.run(load_only=True) + assert summary.nodes["dbl"].not_stored == 2 + assert loaded.nodes["dbl"].as_table().num_rows == 0 # --------------------------------------------------------------------------- @@ -801,7 +817,8 @@ def test_loaded_function_node_cannot_execute(saved: pathlib.Path) -> None: _ = job.nodes["dbl"].execute(job.nodes["dsrc"]) assert str(excinfo.value) == ( "FunctionJobNode 'dbl' was loaded by PipelineJob.load and has no function " - "pod, so it cannot compute; read its stored results with as_source()." + "pod, so it cannot compute; fill it from the store with run(load_only=True), " + "or read its stored results with as_source()." ) diff --git a/tests/test_pipeline/loaded_job/test_loaded_load_only.py b/tests/test_pipeline/loaded_job/test_loaded_load_only.py new file mode 100644 index 000000000..a313ebe7f --- /dev/null +++ b/tests/test_pipeline/loaded_job/test_loaded_load_only.py @@ -0,0 +1,166 @@ +"""A loaded job is empty until ``loaded.run(load_only=True)`` fills it (ITL-1095). + +Developer, November 13, 2026: "the initially loaded job is empty", and a load-only run +fills the nodes from what is stored. A loaded job has no function pods, so it can never +compute; with its sources available (rebuilt from files, or bound), a load-only run reads +them, runs the rebuilt operators, and gives each function node the results its store +holds for those inputs. ``loaded.run()`` without ``load_only`` still raises. +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from typing import Any + +import pyarrow as pa +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.core.operators import Join +from orcapod.core.streams import ViewStatus +from orcapod.databases import InMemoryArrowDatabase +from orcapod.sources import ArrowTableSource, CSVSource, DictSource +from orcapod.types import CacheMode + +JOB = "loaded_load_only" + + +@function_pod(output_keys="dbl") +def double(x: int) -> int: + """Double *x*.""" + return 2 * x + + +@function_pod(output_keys="cc") +def increment(c: int) -> int: + """Add one to *c*.""" + return c + 1 + + +def _dict_source() -> DictSource: + return DictSource( + [{"id": 0, "x": 5}, {"id": 1, "x": 10}], tag_columns=["id"], source_id="dsrc" + ) + + +def _arrow_source() -> ArrowTableSource: + table = pa.table( + {"id": [0, 1], "y": [3, 4]}, + schema=pa.schema( + [ + pa.field("id", pa.int64(), nullable=False), + pa.field("y", pa.int64(), nullable=False), + ] + ), + ) + return ArrowTableSource(table, tag_columns=["id"], source_id="asrc") + + +def _save(tmp_path: Path, db: Any, cache_mode: CacheMode | None = None) -> Path: + """Run and save a job: ``dbl`` over an in-memory source joined with another one + (``joined``), and ``inc`` over a CSV source.""" + csv = tmp_path / "c.csv" + csv.write_text("id,c\n0,7\n1,8\n") + job = PipelineJob(name=JOB, database=db) + with job: + Join()(double.pod(_dict_source(), label="dbl"), _arrow_source(), label="joined") + increment.pod( + CSVSource(str(csv), tag_columns=["id"], source_id="csrc"), label="inc" + ) + if cache_mode is not None: + job.nodes["joined"].attach_databases( + pipeline_database=db.at(JOB), cache_mode=cache_mode + ) + job.run() + path = tmp_path / "job.json" + job.save(path) + return path + + +def _snapshot(db: Any) -> dict[tuple[str, ...], int]: + db.flush() + counts: dict[tuple[str, ...], int] = {} + for path in db.list_table_paths(): + records = db.get_all_records(path) + counts[path] = 0 if records is None else records.num_rows + return counts + + +def _bound(path: Path, db: Any) -> PipelineJob: + loaded = PipelineJob.load(path, database=db) + loaded.bind(sources={"dsrc": _dict_source(), "asrc": _arrow_source()}) + return loaded + + +@pytest.mark.parametrize("orchestrator", ["sync", "async"]) +def test_a_load_only_run_fills_a_loaded_job_and_writes_nothing( + tmp_path: Path, orchestrator: str +) -> None: + db = InMemoryArrowDatabase() + path = _save(tmp_path, db) + before = _snapshot(db) + loaded = _bound(path, db) + for label in ("dbl", "joined", "inc"): + assert loaded.nodes[label].as_table().num_rows == 0, label + if orchestrator == "sync": + summary = loaded.run(load_only=True) + else: + summary = asyncio.run(loaded.run_async(load_only=True)) + assert summary.load_only + assert summary.nodes["dbl"].reused == 2 + assert summary.nodes["inc"].reused == 2 + assert _snapshot(db) == before + dbl = {r["id"]: r["dbl"] for r in loaded.nodes["dbl"].as_table().to_pylist()} + assert dbl == {0: 10, 1: 20} + joined = loaded.nodes["joined"].as_table().to_pylist() + assert sorted((r["id"], r["dbl"], r["y"]) for r in joined) == [ + (0, 10, 3), + (1, 20, 4), + ] + assert loaded.nodes["dbl"].head().status is ViewStatus.LOADED_ONLY + + +def test_nodes_below_an_unbound_source_stay_empty(tmp_path: Path) -> None: + db = InMemoryArrowDatabase() + loaded = PipelineJob.load(_save(tmp_path, db), database=db) + summary = loaded.run(load_only=True) + # The CSV source is rebuilt from its file; the in-memory ones are not saved. + assert loaded.nodes["inc"].as_table().num_rows == 2 + assert loaded.nodes["dbl"].as_table().num_rows == 0 + assert loaded.nodes["joined"].as_table().num_rows == 0 + assert "dbl" not in summary.nodes + + +def test_a_loaded_job_still_cannot_compute(tmp_path: Path) -> None: + db = InMemoryArrowDatabase() + loaded = _bound(_save(tmp_path, db), db) + with pytest.raises(RuntimeError, match=r"run\(load_only=True\)"): + loaded.run() + with pytest.raises(RuntimeError, match=r"run\(load_only=True\)"): + loaded.nodes["dbl"].run() + + +def test_a_loaded_node_runs_load_only_with_what_it_depends_on(tmp_path: Path) -> None: + db = InMemoryArrowDatabase() + loaded = _bound(_save(tmp_path, db), db) + view = loaded.nodes["joined"].run(load_only=True) + assert view.as_table().num_rows == 2 + assert loaded.nodes["dbl"].as_table().num_rows == 2 + summary = loaded.nodes["joined"].last_run + assert summary is not None and summary.load_only + + +def test_a_loaded_replay_operator_replays_its_run_in_a_load_only_run( + tmp_path: Path, +) -> None: + db = InMemoryArrowDatabase() + path = _save(tmp_path, db, cache_mode=CacheMode.LOG) + loaded = _bound(path, db) + loaded.nodes["joined"].attach_databases( + pipeline_database=db.at(JOB), cache_mode=CacheMode.REPLAY + ) + before = _snapshot(db) + loaded.run(load_only=True) + assert _snapshot(db) == before + assert loaded.nodes["joined"].as_table().num_rows == 2 From ee3e485eb55ede907a3540b3d20564afbca4264d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 22:16:08 -0800 Subject: [PATCH 24/63] test(objective): a second node runs before it is read (ITL-1095) Co-Authored-By: Claude Opus 5.5 --- superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md | 2 +- test-objective/integration/test_caching_flows.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md b/superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md index 6d8904834..24aae79a1 100644 --- a/superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md +++ b/superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md @@ -1,6 +1,6 @@ # ITL-1095: a job node shows the current run; load-only run -Status: decisions received (see section 2). Implementing in four stages, one commit (or more) each, pushed to the draft PR. +Status: all four stages implemented on draft PR #380 (stage 1 da66b836, stage 2 c90eb966, stage 3 b693b50e, stage 4 3610fd20). Next: wait for #378 to merge, merge main, CI, CodeRabbit. Base: PR #378 branch head `2e9972e0` (ITL-1093). Branch: `eywalker/itl-1095-design-what-a-job-node-shows-the-current`. Rule (developer, November 13, 2026): a job node itself (`as_table`, `iter_data`, dataframe diff --git a/test-objective/integration/test_caching_flows.py b/test-objective/integration/test_caching_flows.py index 6cdca8a8f..ed91d25e4 100644 --- a/test-objective/integration/test_caching_flows.py +++ b/test-objective/integration/test_caching_flows.py @@ -85,6 +85,8 @@ def test_second_run_uses_cache(self): pipeline_database=pipeline_db, result_database=result_db, ) + # A node shows rows only after a run (ITL-1095); this run reads them all. + node2.run() data = list(node2.iter_data()) assert len(data) == 3 From 40c54f0950ceaec176ecd284ce5143cc11ab7841 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 22:26:46 -0800 Subject: [PATCH 25/63] test(arrays): read the v0.2-store jobs after a load-only run (ITL-1095) Co-Authored-By: Claude Opus 5.5 --- tests/test_arrays/test_array_v02_store.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_arrays/test_array_v02_store.py b/tests/test_arrays/test_array_v02_store.py index da43e93be..8706caa41 100644 --- a/tests/test_arrays/test_array_v02_store.py +++ b/tests/test_arrays/test_array_v02_store.py @@ -103,6 +103,10 @@ def test_v02_rows_are_ignored_recomputed_and_kept( # Read through a newly built job: a Delta database object that read a table before # the run does not see the rows the run added to it (not specific to hash labels). stored = build_job(store) + # A node shows rows only after a run (ITL-1095): a load-only run reads them from the + # store and computes nothing. + stored.run(orchestrator=orchestrator, load_only=True) + assert CALLS == ["plus_two"] * len(ARRAYS) rows = [ (t.as_dict()["k"], d.as_dict()["y"]) for t, d in stored.nodes["double"].iter_data() @@ -154,5 +158,8 @@ def test_second_run_reuses_the_rows_of_this_version(store: Path) -> None: assert CALLS == [] stored = build_job(store) + # A node shows rows only after a run (ITL-1095): a load-only run reads them. + stored.run(load_only=True) + assert CALLS == [] for label in ("double", "total", "direct_sum", "plus_one", "after_drop"): assert stored.nodes[label].as_table().num_rows == len(ARRAYS), label From 74a2ddf5f4596a883f09a99dcc0795eabb237f50 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 22:28:40 -0800 Subject: [PATCH 26/63] fix(pipeline): type-narrowing in the node.run() cascade (ITL-1095) Co-Authored-By: Claude Opus 5.5 --- src/orcapod/pipeline/node_run.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/orcapod/pipeline/node_run.py b/src/orcapod/pipeline/node_run.py index bfeae0e43..a7d4b1fd9 100644 --- a/src/orcapod/pipeline/node_run.py +++ b/src/orcapod/pipeline/node_run.py @@ -114,20 +114,26 @@ def run_with_dependencies(target: Any, *, load_only: bool) -> RunSummary: held: dict[Any, list[Any]] = {} seen: list[Any] = [] + def is_current(node: Any) -> bool: + """Whether ``node`` holds a run this run does not redo.""" + return isinstance(node, RunHoldingNode) and node.holds_current_run( + load_only=load_only + ) + + def is_source(node: Any) -> bool: + """Whether ``node`` is a job's source slot: read by the run, never held.""" + return isinstance(node, SourceJobNode) + def visit(node: Any) -> None: """Add ``node`` and, unless it is held or a source, what it depends on.""" if any(node is other for other in seen): return seen.append(node) dag.add_node(node) - if ( - node is not target - and isinstance(node, RunHoldingNode) - and node.holds_current_run(load_only=load_only) - ): + if node is not target and is_current(node): held[node] = list(node.iter_data()) return - if isinstance(node, SourceJobNode): + if is_source(node): return if not _is_job_node(node): # A stream outside the job: its rows, read as the node's run reads them. From 32bb76639b5126e3d8d510084cc96e90eca647b9 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 22:34:39 -0800 Subject: [PATCH 27/63] fix(function_node): an async run whose output close fails defines no rows (ITL-1093) Addresses the third CodeRabbit review of PR #378 (outside-diff finding). Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/function_node.py | 7 ++- .../test_job_node_rows_after_run.py | 63 +++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 1f81a2f34..54f83f8b4 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -3694,4 +3694,9 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: self.clear_cache() raise finally: - await output.close() + try: + await output.close() + except BaseException: + # A close that raises or is cancelled fails the run: no rows. + self.clear_cache() + raise diff --git a/tests/test_pipeline/test_job_node_rows_after_run.py b/tests/test_pipeline/test_job_node_rows_after_run.py index e45ff97bc..c31e7c487 100644 --- a/tests/test_pipeline/test_job_node_rows_after_run.py +++ b/tests/test_pipeline/test_job_node_rows_after_run.py @@ -475,3 +475,66 @@ def on_node_end(self, node_label: str, node_hash: str) -> None: job.run(orchestrator=_orchestrator(kind), observer=FailsAtNodeEnd()) # Every stored row (the 6-row run's), read from the store. assert _values(node, lambda r: r["k"]) == [0, 1, 2, 3, 4, 5] + + +class TestAsyncOutputCloseFails: + """An async run whose output cannot be closed defines no rows (ITL-1093).""" + + def test_failed_close_leaves_no_run_rows(self) -> None: + """A close that raises after every row was sent leaves the node with no run.""" + import asyncio + + import pyarrow as pa + + from orcapod.channels import Channel + from orcapod.core.nodes import FunctionJobNode + from orcapod.core.streams import ArrowTableStream + + class CloseFails: + """An output writer that keeps what it is sent and raises on close.""" + + def __init__(self) -> None: + """Start with no rows.""" + self.sent: list[Any] = [] + + async def send(self, item: Any) -> None: + """Keep ``item``.""" + self.sent.append(item) + + async def close(self) -> None: + """Raise, as a closed or broken channel would.""" + raise OSError("output cannot be closed") + + schema = pa.schema( + [ + pa.field("k", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] + ) + table = pa.Table.from_arrays( + [pa.array([0, 1], pa.int64()), pa.array([5, 6], pa.int64())], + schema=schema, + ) + node = FunctionJobNode( + inc.pod, # pyright: ignore[reportArgumentType] # the decorator's pod type is not declared as FunctionPodProtocol + ArrowTableStream(table, tag_columns=["k"]), + pipeline_database=InMemoryArrowDatabase(), + result_database=InMemoryArrowDatabase(), + ) + output = CloseFails() + + async def run() -> None: + """Run the node over its upstream's rows into ``output``.""" + inputs = Channel(buffer_size=16) + # The input stream is the node's own upstream (a test of async_execute). + for item in node.upstreams[0].iter_data(): + await inputs.writer.send(item) + await inputs.writer.close() + await node.async_execute(inputs.reader, output) # pyright: ignore[reportArgumentType] # a test double of the writer + + with pytest.raises(OSError, match="cannot be closed"): + asyncio.run(run()) + assert len(output.sent) == 2 + # No run is held: the rows are read from the store. + assert node.head(10).origin == "stored" + assert _values(node, lambda r: (r["k"], r["y"])) == [(0, 6), (1, 7)] From 2846dde605236c8845d82220956b388167ad61ac Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 01:30:58 -0800 Subject: [PATCH 28/63] feat(side_effects): a side-effect job node follows the job node state rule (ITL-1095) After #377 (ITL-1091) a SideEffectJobNode holds the rows its last run passed on; it now mixes in RunHoldingNode: stale before a run, transitive staleness, a current node delivers nothing on run() (clear_cache() to deliver again), run() cascades as every job node's does (shared run_job_node), and a load-only run passes its rows on, holds them as load-only and delivers nothing. The orchestrators no longer special-case side-effect nodes in a load-only run. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 7 +- CLAUDE.md | 7 +- changelog.d/ITL-1095.md | 1 + src/orcapod/core/nodes/function_node.py | 57 +++++++--- src/orcapod/core/nodes/staleness.py | 5 +- src/orcapod/pipeline/async_orchestrator.py | 39 +++---- src/orcapod/pipeline/sync_orchestrator.py | 26 ++--- src/orcapod/protocols/node_protocols.py | 2 + src/orcapod/side_effects.py | 103 +++++++++++++++--- .../test_side_effect_job_node_reads.py | 13 ++- tests/test_core/streams/test_run.py | 7 +- tests/test_pipeline/test_node_run_cascade.py | 28 +++++ 12 files changed, 213 insertions(+), 82 deletions(-) diff --git a/.zed/rules b/.zed/rules index a9d2c6d4f..dd15f3ebd 100644 --- a/.zed/rules +++ b/.zed/rules @@ -804,9 +804,10 @@ data. The rules: A loaded job (PipelineJob.load) runs only with load_only=True. - Empty data → ArrowTableSource raises ValueError("Table is empty"). - A side-effect job node delivers only when it runs (developer, November 13, 2026, ITL-1091): - job.run(), node.run() (the job run's work for that node: with track_completion only - what the invocation log does not record, without it every row again), execute, - async_execute. Every read (iter_data, as_table, dataframes, flow, head, preview, a + job.run(), node.run() (state-driven, ITL-1095: a node holding a current run delivers + nothing, so clear_cache() it first to deliver again; otherwise the job run's work for that + node: with track_completion only what the invocation log does not record, without it every + row), execute, async_execute; a load-only run delivers nothing. Every read (iter_data, as_table, dataframes, flow, head, preview, a downstream node's read) returns the rows its last run in this process passed on, held in memory (SideEffectJobNode._held_rows), and delivers nothing; with none (no run, a run that raised, or stale by is_stale) a read returns zero rows. Never route a job node's read to diff --git a/CLAUDE.md b/CLAUDE.md index 54b615442..1ada669b9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -846,9 +846,10 @@ data. The rules: A loaded job (`PipelineJob.load`) runs only with `load_only=True`. - Empty data → `ArrowTableSource` raises `ValueError("Table is empty")`. - A side-effect job node delivers only when it runs (developer, November 13, 2026, ITL-1091): - `job.run()`, `node.run()` (the job run's work for that node: with `track_completion` only - what the invocation log does not record, without it every row again), `execute`, - `async_execute`. Every read (`iter_data`, `as_table`, dataframes, `flow`, `head`, preview, a + `job.run()`, `node.run()` (state-driven, ITL-1095: a node holding a current run delivers + nothing, so `clear_cache()` it first to deliver again; otherwise the job run's work for that + node: with `track_completion` only what the invocation log does not record, without it every + row), `execute`, `async_execute`; a load-only run delivers nothing. Every read (`iter_data`, `as_table`, dataframes, `flow`, `head`, preview, a downstream node's read) returns the rows its last run in this process passed on, held in memory (`SideEffectJobNode._held_rows`), and delivers nothing; with none (no run, a run that raised, or stale by `is_stale`) a read returns zero rows. Never route a job node's read to diff --git a/changelog.d/ITL-1095.md b/changelog.d/ITL-1095.md index 543769b5f..7675b59a9 100644 --- a/changelog.d/ITL-1095.md +++ b/changelog.d/ITL-1095.md @@ -4,6 +4,7 @@ - A function or operator job node that has not run in this process is stale (`is_stale` is `True`; it has no modified time), and a job node is stale also when any function or operator job node it depends on is. Binding a source to a job's source slot (`job.bind(sources=...)`) marks the slot modified, so every node fed by it is stale and empty until it runs again. - A loaded operator node no longer raises from `as_table()` or `iter_data()`: it is empty, like every node that has not run. Its stored view (`as_source()`, `get_all_records()`) raises instead when the job file does not record its run's content hash and the store holds nothing under the hash the loaded job computes (ITL-1043). - A job node's `run()` is state-driven and runs what the node depends on (developer, November 13, 2026; ITL-1095). A node that holds a completed run and is not stale runs nothing: `run()` returns the rows it holds. Otherwise every job node it depends on that holds no current run runs first, each once, as `job.run()` would run that part of the job, then the node; the run's writes are committed when it ends (before, `node.run()` read an upstream node's stored rows, or nothing, and left its last pipeline record and a `LOG` operator's output uncommitted). `run()` of a node whose function pod is a placeholder, or that was loaded without one, raises unless `load_only=True`; a `CACHE_ONLY` node's `run()` raises (its input is unavailable): read its stored results with `as_source()`. A job node's `run()` takes one keyword-only argument, `load_only`; every other stream's `run()` still takes none. +- A side-effect job node follows the same rule: it is stale before a run, and while it holds a current run its `run()` (or a downstream node's `run()`) delivers nothing; to deliver again without `track_completion`, call `clear_cache()` on it first (`job.run()` still delivers as before). A load-only run passes its rows on and delivers nothing. - If the commit at the end of `job.run()` or `node.run()` fails, every function and operator node of the run is cleared (not run, stale) before the error propagates. ### Added diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index d2758686b..8a3e93ed3 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -274,6 +274,44 @@ async def close(self) -> None: # --------------------------------------------------------------------------- +def run_job_node( + node: Any, + *, + load_only: bool, + run_alone: Callable[[], Any], + caller: Any, +) -> None: + """Run a job node as its ``run()`` does (developer, November 13, 2026; ITL-1095). + + Nothing runs when the node holds a current run (``holds_current_run``). Otherwise + a node that depends on job nodes runs with them (``run_with_dependencies``) and one + built over streams outside a job runs alone (``run_alone``). The run's summary, if + it gives one, is kept as the node's ``last_run``, and failed rows warn at the + caller's line, as ``job.run()`` warns. + + Args: + node: The job node (a ``RunHoldingNode``). + load_only: Whether to load stored results only. + run_alone: Runs the node over its input when no job node feeds it; returns its + ``RunSummary`` or ``None``. + caller: The frame of the ``run()`` caller. + """ + from orcapod.pipeline.node_run import depends_on_job_nodes, run_with_dependencies + from orcapod.pipeline.run_summary import warn_row_failures + + if node.holds_current_run(load_only=load_only): + return + if depends_on_job_nodes(node): + summary = run_with_dependencies(node, load_only=load_only) + else: + summary = run_alone() + if summary is None: + return + node._last_run = summary + if not summary.ok: + warn_row_failures(summary, caller) + + class FunctionNodeBase(StreamBase): """Shared base for ``FunctionNode`` and ``FunctionJobNode``. @@ -3447,11 +3485,6 @@ def run(self, *, load_only: bool = False) -> StreamView: ``load_only`` is false; or if its input is unavailable (``CACHE_ONLY``): read its stored rows with ``as_source()``. """ - from orcapod.pipeline.node_run import ( - depends_on_job_nodes, - run_with_dependencies, - ) - from orcapod.pipeline.run_summary import warn_row_failures from orcapod.pipeline.serialization import LoadStatus status = self._load_status @@ -3466,14 +3499,12 @@ def run(self, *, load_only: bool = False) -> StreamView: self._function_pod is None or status == LoadStatus.READ_ONLY ): raise self._cannot_compute_error() - if not self.holds_current_run(load_only=load_only): - if depends_on_job_nodes(self): - summary = run_with_dependencies(self, load_only=load_only) - else: - summary = self._run_alone(load_only=load_only) - self._last_run = summary - if not summary.ok: - warn_row_failures(summary, sys._getframe(1)) # pyright: ignore[reportPrivateUsage] # the caller's line is where the warning is shown + run_job_node( + self, + load_only=load_only, + run_alone=lambda: self._run_alone(load_only=load_only), + caller=sys._getframe(1), # pyright: ignore[reportPrivateUsage] # the caller's line is where a failure warning is shown + ) return self._run_view() def _run_alone(self, *, load_only: bool) -> Any: diff --git a/src/orcapod/core/nodes/staleness.py b/src/orcapod/core/nodes/staleness.py index 841a96fd0..fc461a009 100644 --- a/src/orcapod/core/nodes/staleness.py +++ b/src/orcapod/core/nodes/staleness.py @@ -15,9 +15,8 @@ class RunHoldingNode: """Mixin of the job nodes that hold the rows of their current run. - ``FunctionJobNode`` and ``OperatorJobNode`` mix it in ahead of their stream base, so - their ``is_stale`` covers their whole ancestry. Side-effect job nodes do not yet: - their own ``is_stale`` is redefined by ITL-1091. + ``FunctionJobNode``, ``OperatorJobNode`` and ``SideEffectJobNode`` mix it in ahead of + their stream base, so their ``is_stale`` covers their whole ancestry. """ # The summary of the node's last ``run()`` (with the nodes it ran first); a class diff --git a/src/orcapod/pipeline/async_orchestrator.py b/src/orcapod/pipeline/async_orchestrator.py index 592d6fdb7..6cc442198 100644 --- a/src/orcapod/pipeline/async_orchestrator.py +++ b/src/orcapod/pipeline/async_orchestrator.py @@ -31,7 +31,6 @@ ) if TYPE_CHECKING: - from orcapod.channels import WritableChannel from orcapod.pipeline.dag import GraphProtocol from orcapod.protocols.core_protocols import DataProtocol, TagProtocol from orcapod.protocols.observability_protocols import ExecutionObserverProtocol @@ -301,20 +300,19 @@ async def _run_async( "SideEffectNode expects exactly 1 upstream, got 0" ) input_reader = edge_readers[(predecessors[0], node)] - if load_only: - # Nothing is delivered in a load-only run: rows pass on. - _start_node(tg, node, _pass_through(input_reader, writer)) - else: - _start_node( - tg, - node, - node.async_execute( - [input_reader], - writer, - observer=effective_observer, - run_id=run_id, - ), - ) + # In a load-only run the node passes its rows on and delivers + # nothing. + _start_node( + tg, + node, + node.async_execute( + [input_reader], + writer, + observer=effective_observer, + run_id=run_id, + load_only=load_only, + ), + ) else: raise TypeError( f"Unknown node type: {getattr(node, 'node_type', None)!r}" @@ -610,17 +608,6 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: return NoOpLogger() -async def _pass_through( - reader: Any, writer: WritableChannel[tuple[TagProtocol, DataProtocol]] -) -> None: - """Forward every row of ``reader`` to ``writer``, then close ``writer``.""" - try: - async for item in reader: - await writer.send(item) - finally: - await writer.close() - - def _start_node( tg: asyncio.TaskGroup, node: Any, work: Coroutine[Any, Any, None] ) -> None: diff --git a/src/orcapod/pipeline/sync_orchestrator.py b/src/orcapod/pipeline/sync_orchestrator.py index bcca4f106..a02723c57 100644 --- a/src/orcapod/pipeline/sync_orchestrator.py +++ b/src/orcapod/pipeline/sync_orchestrator.py @@ -75,7 +75,7 @@ def run( load_only: Fill the nodes from what is stored and compute nothing (ITL-1095): function nodes load their inputs' stored results and skip the others, operators run without storing, and a side-effect node - passes its rows on without being called (nothing is delivered). + passes its rows on and delivers nothing. held_rows: Nodes of ``graph`` that are not run: each one's rows (the rows of its current run, or of a stream outside the job) feed its successors as if it had just run. Used by ``node.run()``, which runs @@ -128,19 +128,17 @@ def run( ) elif is_side_effect_node(node): upstream_buf = self._gather_upstream(node, graph, buffers) - if load_only: - # Nothing is delivered in a load-only run: the rows pass on. - buffers[node] = list(upstream_buf) - else: - upstream_node = list(graph.predecessors(node))[0] - input_stream = self._materialize_as_stream( - upstream_buf, upstream_node - ) - buffers[node] = node.execute( - input_stream, - observer=effective_observer, - run_id=run_id, - ) + upstream_node = list(graph.predecessors(node))[0] + input_stream = self._materialize_as_stream( + upstream_buf, upstream_node + ) + # In a load-only run the node passes its rows on and delivers nothing. + buffers[node] = node.execute( + input_stream, + observer=effective_observer, + run_id=run_id, + load_only=load_only, + ) else: raise TypeError( f"Unknown node type: {getattr(node, 'node_type', None)!r}" diff --git a/src/orcapod/protocols/node_protocols.py b/src/orcapod/protocols/node_protocols.py index c33a71c84..183d3a4ae 100644 --- a/src/orcapod/protocols/node_protocols.py +++ b/src/orcapod/protocols/node_protocols.py @@ -160,6 +160,7 @@ def execute( *, observer: "ExecutionObserverProtocol | None" = None, run_id: str | None = None, + load_only: bool = False, ) -> "list[tuple[TagProtocol, DataProtocol]]": ... async def async_execute( @@ -169,6 +170,7 @@ async def async_execute( *, observer: "ExecutionObserverProtocol | None" = None, run_id: str | None = None, + load_only: bool = False, ) -> None: ... def attach_databases( diff --git a/src/orcapod/side_effects.py b/src/orcapod/side_effects.py index fb9ad3d2e..6da628ced 100644 --- a/src/orcapod/side_effects.py +++ b/src/orcapod/side_effects.py @@ -18,7 +18,8 @@ from typing_extensions import override from orcapod.core.base import TraceableBase -from orcapod.core.nodes.function_node import _build_record_id_preimage +from orcapod.core.nodes.staleness import RunHoldingNode +from orcapod.core.nodes.function_node import _build_record_id_preimage, run_job_node from orcapod.core.streams.base import ( StreamBase, check_head_size, @@ -915,7 +916,7 @@ def node_uri(self) -> tuple[str, ...]: # --------------------------------------------------------------------------- -class SideEffectJobNode(SideEffectNode): +class SideEffectJobNode(RunHoldingNode, SideEffectNode): """DB-backed execution node for side-effect pods. Created at pipeline compile time by ``PipelineJob``. Receives a @@ -950,6 +951,28 @@ def __init__( # The rows the last completed run passed on, in the order it passed them; # ``None`` when no run in this process completed (ITL-1091). self._held_rows: list[tuple[TagProtocol, DataProtocol]] | None = None + # Whether those rows came from a load-only run, which delivers nothing. + self._run_load_only = False + # A node that has not run in this process holds nothing and is stale (ITL-1095). + self._set_modified_time(None) + + def _holds_run(self, *, load_only: bool) -> bool: + """Whether the node holds a run's rows; a load-only run's only for ``load_only``. + + A load-only run delivers nothing, so a full run does not take its rows as the + result of a delivery. + """ + return self._held_rows is not None and (load_only or not self._run_load_only) + + def clear_cache(self) -> None: + """Drop the rows the last run passed on: the node is stale and reads empty. + + Its next ``run()`` (or job run) delivers again what its invocation log does not + record (every row without ``track_completion``) (ITL-1095). + """ + self._held_rows = None + self._run_load_only = False + self._set_modified_time(None) def _refuse_log_of_another_data_context(self) -> None: """Refuse an invocation log holding deliveries of another data context. @@ -1036,6 +1059,7 @@ def execute( *, observer: ExecutionObserverProtocol | None = None, run_id: str | None = None, + load_only: bool = False, ) -> list[tuple[TagProtocol, DataProtocol]]: """Execute side-effect delivery for all rows in ``input_stream``. @@ -1043,6 +1067,8 @@ def execute( input_stream: Stream of ``(tag, data)`` pairs to process. observer: Optional execution observer (currently unused). run_id: Pipeline run identifier forwarded from the orchestrator. + load_only: A load-only run (ITL-1095): every row passes on and nothing is + delivered or logged. The node keeps the rows it passed on, which every later read returns without delivering (ITL-1091). A run that raises leaves the node with no rows. @@ -1055,6 +1081,10 @@ def execute( logged under another data context; nothing is delivered then. """ self._held_rows = None + if load_only: + rows = list(input_stream.iter_data()) + self._hold(rows, load_only=True) + return rows self._refuse_log_of_another_data_context() results: list[tuple[TagProtocol, DataProtocol]] = [] for tag, data in input_stream.iter_data(): @@ -1077,13 +1107,20 @@ def execute( self._hold(results) return results - def _hold(self, rows: list[tuple[TagProtocol, DataProtocol]]) -> None: + def _hold( + self, rows: list[tuple[TagProtocol, DataProtocol]], *, load_only: bool = False + ) -> None: """Keep the rows a completed run passed on and stamp the node as modified. Stamped after the upstream nodes, which stamp at the end of their own run, so the rows are current until an upstream changes again (``is_stale``). + + Args: + rows: The rows the run passed on. + load_only: Whether the run was load-only (it delivered nothing). """ self._held_rows = rows + self._run_load_only = load_only self._update_modified_time() def _current_rows(self) -> list[tuple[TagProtocol, DataProtocol]] | None: @@ -1150,18 +1187,25 @@ def head( ) @override - def run(self) -> StreamView: + def run(self, *, load_only: bool = False) -> StreamView: """Run the node as ``job.run()`` runs it and return a ``StreamView`` of its rows. - Reads the input and delivers through the invocation log, as a job run does: - with ``track_completion`` only the rows the log does not record as delivered are - delivered; without it every row is delivered again, as a second ``job.run()`` - delivers it. With no database attached nothing is logged and every row is - delivered. The node then holds the rows it passed on, which ``as_table()`` and - every other read return without delivering. The invocation log is committed - before ``run()`` returns, as at the end of a job run; after a failed delivery - the log rows of the deliveries that succeeded are committed too, and the - delivery's error is raised. + State-driven, as every job node's ``run()`` (developer, November 13, 2026; + ITL-1095): a node that holds the rows of a completed, non-stale run delivers + nothing and returns them; call ``clear_cache()`` first to deliver again. + Otherwise the job nodes it depends on that hold no current run run first, and + the node reads its input and delivers through the invocation log, as a job run + does: with ``track_completion`` only the rows the log does not record as + delivered; without it every row. With no database attached nothing is logged + and every row is delivered. The node then holds the rows it passed on, which + ``as_table()`` and every other read return without delivering. The run's writes + (the invocation log included) are committed before ``run()`` returns; after a + failed delivery the log rows of the deliveries that succeeded are committed too, + and the delivery's error is raised. + + Args: + load_only: Deliver nothing: the rows pass on, and the nodes run first load + their stored results only (as ``job.run(load_only=True)``). Returns: A ``StreamView`` of the rows this run passed on. @@ -1170,9 +1214,21 @@ def run(self) -> StreamView: SideEffectLogDataContextError: As for ``execute``. StoreWriteError: If committing the invocation log fails. """ + import sys + + run_job_node( + self, + load_only=load_only, + run_alone=lambda: self._run_alone(load_only=load_only), + caller=sys._getframe(1), # pyright: ignore[reportPrivateUsage] # the caller's line is where a failure warning is shown + ) + return self._view_of_rows(self.as_table(), "held") + + def _run_alone(self, *, load_only: bool) -> None: + """Run this node over an input stream that is not a job node; commit its log.""" database = self._pipeline_database try: - _ = self.execute(self._input_stream) + _ = self.execute(self._input_stream, load_only=load_only) except Exception as run_error: if database is not None: flush_best_effort(run_error, database) @@ -1184,12 +1240,11 @@ def run(self) -> StreamView: except StoreReadError: # A stored table could not be read: report the read failure itself. # The run raised, so it leaves the node no rows. - self._held_rows = None + self.clear_cache() raise except Exception as exc: - self._held_rows = None + self.clear_cache() raise StoreWriteError(database.base_path, flush=True) from exc - return self._view_of_rows(self.as_table(), "held") # ------------------------------------------------------------------ # Async execution @@ -1202,6 +1257,7 @@ async def async_execute( *, observer: ExecutionObserverProtocol | None = None, run_id: str | None = None, + load_only: bool = False, ) -> None: """Async side-effect delivery with semaphore-bounded concurrency. @@ -1222,6 +1278,8 @@ async def async_execute( output: Writable channel for pass-through ``(tag, data)`` pairs. observer: Optional execution observer (currently unused). run_id: Pipeline run identifier from the orchestrator. + load_only: A load-only run (ITL-1095): every row passes on and nothing is + delivered or logged. Raises: ValueError: If ``inputs`` does not contain exactly one channel. @@ -1237,6 +1295,17 @@ async def async_execute( ) self._held_rows = None + if load_only: + try: + rows: list[tuple[TagProtocol, DataProtocol]] = [] + async for item in inputs[0]: + rows.append(item) + await output.send(item) + # Held and stamped before the output closes (ITL-1042). + self._hold(rows, load_only=True) + finally: + await output.close() + return # This run's rows, in the order they were sent downstream. passed_on: list[tuple[TagProtocol, DataProtocol]] = [] # Makes recording a row and sending it one step: a send blocked by a full channel diff --git a/tests/test_core/side_effect_pod/test_side_effect_job_node_reads.py b/tests/test_core/side_effect_pod/test_side_effect_job_node_reads.py index 47e803a67..cb272bca9 100644 --- a/tests/test_core/side_effect_pod/test_side_effect_job_node_reads.py +++ b/tests/test_core/side_effect_pod/test_side_effect_job_node_reads.py @@ -184,13 +184,19 @@ def test_node_run_delivers_what_the_log_does_not_record( @pytest.mark.parametrize("orchestrator", ["sync", "async"]) -def test_node_run_without_track_completion_delivers_every_row_as_a_job_run_does( +def test_node_run_without_track_completion_delivers_again_only_after_clear_cache( orchestrator: Orchestrator, ) -> None: - """Without ``track_completion``, ``node.run()`` delivers every row again, as ``job.run()`` does.""" + """Without ``track_completion``, a node that holds a current run delivers nothing on + ``node.run()`` (ITL-1095); after ``clear_cache()`` it delivers every row again, as + ``job.run()`` does.""" job, sent = _job(track_completion=False) _run(job, orchestrator) view = job.nodes["delivered"].run() + assert len(sent) == ROWS + assert len(view) == ROWS + job.nodes["delivered"].clear_cache() + view = job.nodes["delivered"].run() assert len(sent) == 2 * ROWS assert len(view) == ROWS _run(job, orchestrator) @@ -319,6 +325,9 @@ def test_rows_held_from_a_run_are_dropped_when_an_upstream_changes( _run(job, orchestrator) node = job.nodes["delivered"] assert len(node.as_table()) == ROWS + # The upstream runs again: a node that holds a current run runs nothing (ITL-1095), + # so it is cleared first. + job.nodes["up"].clear_cache() _ = job.nodes["up"].run() for name, read in _reads(node).items(): _ = read() diff --git a/tests/test_core/streams/test_run.py b/tests/test_core/streams/test_run.py index 5df3b027a..c11660fcf 100644 --- a/tests/test_core/streams/test_run.py +++ b/tests/test_core/streams/test_run.py @@ -117,7 +117,12 @@ def test_every_stream_class_has_run_without_positional_arguments(cls: type) -> N assert callable(run), f"{cls.__qualname__} has no run()" parameters = inspect.signature(run).parameters names = list(parameters) - if cls.__name__ in ("FunctionJobNode", "OperatorJobNode", "SourceJobNode"): + if cls.__name__ in ( + "FunctionJobNode", + "OperatorJobNode", + "SideEffectJobNode", + "SourceJobNode", + ): assert names == ["self", "load_only"], f"{cls.__qualname__}.run{names}" load_only = parameters["load_only"] assert load_only.kind is inspect.Parameter.KEYWORD_ONLY diff --git a/tests/test_pipeline/test_node_run_cascade.py b/tests/test_pipeline/test_node_run_cascade.py index 4d518933b..5fd93ffff 100644 --- a/tests/test_pipeline/test_node_run_cascade.py +++ b/tests/test_pipeline/test_node_run_cascade.py @@ -288,6 +288,34 @@ def test_a_load_only_run_that_found_everything_stays_loaded_only(self) -> None: class TestSideEffects: + def test_a_side_effect_node_follows_the_state_rule(self) -> None: + """Stale before a run; while it holds a current run nothing re-delivers it.""" + job = _build(ROWS, InMemoryArrowDatabase(), with_tap=True) + tap = job.nodes["tap"] + assert tap.is_stale + _ = job.nodes["summed"].run() + assert len(DELIVERED) == 6 + assert tap.holds_current_run() + _ = job.nodes["summed"].run() + _ = tap.run() + assert len(DELIVERED) == 6 + tap.clear_cache() + assert job.nodes["summed"].is_stale + _ = job.nodes["summed"].run() + assert len(DELIVERED) == 12 + + def test_a_load_only_run_of_a_side_effect_node_holds_its_rows(self) -> None: + db = InMemoryArrowDatabase() + _build(ROWS, db, with_tap=True).run() + DELIVERED.clear() + job = _build(ROWS, db, with_tap=True) + _ = job.nodes["summed"].run(load_only=True) + tap = job.nodes["tap"] + assert DELIVERED == [] + assert tap.as_table().num_rows == 6 + assert tap.holds_current_run(load_only=True) + assert not tap.holds_current_run() + def test_a_full_cascade_delivers_a_load_only_cascade_does_not(self) -> None: db = InMemoryArrowDatabase() job = _build(ROWS, db, with_tap=True) From 42817553223de286fc74d2e923c1ba1a470a0412 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 01:43:50 -0800 Subject: [PATCH 29/63] test(pipeline): failing tests: job.run() follows the state rule (ITL-1095) Co-Authored-By: Claude Opus 5.5 --- .../test_pipeline/test_job_run_state_rule.py | 210 ++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 tests/test_pipeline/test_job_run_state_rule.py diff --git a/tests/test_pipeline/test_job_run_state_rule.py b/tests/test_pipeline/test_job_run_state_rule.py new file mode 100644 index 000000000..32761f012 --- /dev/null +++ b/tests/test_pipeline/test_job_run_state_rule.py @@ -0,0 +1,210 @@ +"""``job.run()`` follows the state rule of ``node.run()`` (ITL-1095). + +Developer, November 14, 2026 ("let's align now"): ``job.run()`` and ``run_async()`` apply +the same rule as ``node.run()``. A node that holds a current completed run (not stale; for +a full run, not filled by a load-only run) is not run again: its rows feed the nodes after +it. Never-run, stale and (for a full run) load-only-filled nodes run. So a second +``job.run()`` of an unchanged job computes nothing, reads and writes nothing in the store +and delivers no side effect. +""" + +from __future__ import annotations + +import asyncio +from typing import Any +from unittest import mock + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.errors import StoreWriteError +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.side_effects import SideEffectPod, tap_pod +from orcapod.sources import DictSource + +CALLS: list[str] = [] +DELIVERED: list[int] = [] + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + """Return ``x + 1``.""" + CALLS.append("inc") + return x + 1 + + +@function_pod(output_keys="total") +def total(y: list[int]) -> int: + """Return the sum of a group's values.""" + CALLS.append("total") + return sum(y) + + +def _deliver(y: int, ctx: Any) -> None: + """A side effect: record the delivered value.""" + DELIVERED.append(y) + + +ROWS = [{"g": i % 2, "k": i, "x": 10 * i} for i in range(6)] +ORCHESTRATORS = {"sync": SyncPipelineOrchestrator, "async": AsyncPipelineOrchestrator} + + +@pytest.fixture(autouse=True) +def _reset() -> None: + CALLS.clear() + DELIVERED.clear() + + +def _source(data: list[dict[str, int]]) -> DictSource: + return DictSource(data, tag_columns=["g", "k"], source_id="s") + + +def _build(data: list[dict[str, int]], db: Any) -> PipelineJob: + """``rowwise`` -> ``tap`` (side effect, no completion tracking) -> ``grouped`` -> + ``summed``.""" + job = PipelineJob(name="j", database=db) + with job: + rows = inc.pod(_source(data), label="rowwise") + tap = tap_pod(_deliver) + assert isinstance(tap, SideEffectPod) + tapped: Any = tap.process(rows, label="tap") + grouped = tapped.group_by("g", label="grouped") + total.pod(grouped, label="summed") + return job + + +def _run(job: PipelineJob, orchestrator: str, **kwargs: Any) -> Any: + return job.run(orchestrator=ORCHESTRATORS[orchestrator](), **kwargs) + + +def _totals(job: PipelineJob) -> list[tuple[int, int]]: + return sorted( + (r["g"], r["total"]) for r in job.nodes["summed"].as_table().to_pylist() + ) + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_a_second_run_of_an_unchanged_job_does_nothing(orchestrator: str) -> None: + job = _build(ROWS, InMemoryArrowDatabase()) + _run(job, orchestrator) + assert len(DELIVERED) == 6 + CALLS.clear() + DELIVERED.clear() + with ( + mock.patch.object( + InMemoryArrowDatabase, + "get_all_records", + side_effect=AssertionError("store read"), + ), + mock.patch.object( + InMemoryArrowDatabase, + "add_records", + side_effect=AssertionError("store write"), + ), + mock.patch.object( + DictSource, "iter_data", side_effect=AssertionError("source read") + ), + ): + summary = _run(job, orchestrator) + assert CALLS == [] + assert DELIVERED == [] + assert _totals(job) == [(0, 63), (1, 93)] + rowwise = summary.nodes["rowwise"] + assert (rowwise.computed, rowwise.reused, rowwise.held) == (0, 0, 6) + assert summary.nodes["summed"].held == 2 + assert "rowwise: 6 rows: held from the previous run" in repr(summary) + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_a_cleared_node_runs_again_with_the_nodes_below_it(orchestrator: str) -> None: + """Clearing ``summed`` runs it alone, from the rows ``grouped`` holds.""" + job = _build(ROWS, InMemoryArrowDatabase()) + _run(job, orchestrator) + CALLS.clear() + DELIVERED.clear() + job.nodes["summed"].clear_cache() + summary = _run(job, orchestrator) + assert CALLS == [] # its results are stored: read, not computed + assert DELIVERED == [] + assert summary.nodes["summed"].reused == 2 + assert summary.nodes["rowwise"].held == 6 + assert _totals(job) == [(0, 63), (1, 93)] + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_clearing_a_side_effect_node_delivers_it_again(orchestrator: str) -> None: + job = _build(ROWS, InMemoryArrowDatabase()) + _run(job, orchestrator) + DELIVERED.clear() + job.nodes["tap"].clear_cache() + _run(job, orchestrator) + assert len(DELIVERED) == 6 + assert CALLS.count("inc") == 6 # only the first run computed + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_a_rebound_source_runs_every_node_below_it(orchestrator: str) -> None: + job = _build(ROWS[:3], InMemoryArrowDatabase()) + _run(job, orchestrator) + CALLS.clear() + job.bind(sources={next(iter(job.sources)): _source(ROWS)}) + _run(job, orchestrator) + assert CALLS.count("inc") == 3 + assert _totals(job) == [(0, 63), (1, 93)] + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_a_full_run_after_a_load_only_run_runs_the_load_only_nodes( + orchestrator: str, +) -> None: + db = InMemoryArrowDatabase() + _run(_build(ROWS[:3], db), orchestrator) + CALLS.clear() + DELIVERED.clear() + job = _build(ROWS, db) + _run(job, orchestrator, load_only=True) + assert CALLS == [] and DELIVERED == [] + _run(job, orchestrator) + assert CALLS.count("inc") == 3 + assert len(DELIVERED) == 6 + assert _totals(job) == [(0, 63), (1, 93)] + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_a_load_only_run_after_a_full_run_does_nothing(orchestrator: str) -> None: + job = _build(ROWS, InMemoryArrowDatabase()) + _run(job, orchestrator) + CALLS.clear() + summary = _run(job, orchestrator, load_only=True) + assert CALLS == [] + assert summary.nodes["rowwise"].held == 6 + + +def test_run_async_follows_the_rule() -> None: + job = _build(ROWS, InMemoryArrowDatabase()) + asyncio.run(job.run_async()) + CALLS.clear() + DELIVERED.clear() + summary = asyncio.run(job.run_async()) + assert CALLS == [] and DELIVERED == [] + assert summary.nodes["summed"].held == 2 + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_a_failed_commit_clears_only_the_nodes_that_ran(orchestrator: str) -> None: + job = _build(ROWS, InMemoryArrowDatabase()) + _run(job, orchestrator) + job.nodes["summed"].clear_cache() + + def fail(self: Any) -> None: + raise OSError("disk full") + + with ( + mock.patch.object(InMemoryArrowDatabase, "flush", fail), + pytest.raises(StoreWriteError), + ): + _run(job, orchestrator) + assert job.nodes["summed"].as_table().num_rows == 0 + assert job.nodes["rowwise"].as_table().num_rows == 6 + assert job.nodes["grouped"].as_table().num_rows == 2 From b1b99445c38ad27949a1e1360503867051890447 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 01:59:51 -0800 Subject: [PATCH 30/63] feat(pipeline)!: job.run() follows the job node state rule (ITL-1095) Developer, November 14, 2026: job.run() and run_async() use the same plan as node.run() (node_run.plan_run): a node that holds a current run is held, not run; its rows feed its successors on both orchestrators (the async one takes held rows too); a source is read only for nodes that run. A second run of an unchanged job computes, reads, writes and delivers nothing; the summary reports held nodes (NodeRunSummary.held: 'held from the previous run'). A failed end-of-run commit clears only the nodes that ran. Tests that ran one job twice to re-run a node now clear that node first. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 5 + CLAUDE.md | 5 + changelog.d/ITL-1095.md | 1 + docs/concepts/function-pods.md | 60 ++++-- src/orcapod/pipeline/async_orchestrator.py | 29 ++- src/orcapod/pipeline/job.py | 44 +++-- src/orcapod/pipeline/node_run.py | 172 +++++++++++++----- src/orcapod/pipeline/run_summary.py | 32 +++- .../test_side_effect_job_node_reads.py | 7 + .../test_pipeline/test_hash_step_combined.py | 7 +- .../test_job_node_rows_after_run.py | 2 + tests/test_pipeline/test_sync_orchestrator.py | 3 + 12 files changed, 281 insertions(+), 86 deletions(-) diff --git a/.zed/rules b/.zed/rules index dd15f3ebd..b2ef8061c 100644 --- a/.zed/rules +++ b/.zed/rules @@ -802,6 +802,11 @@ data. The rules: commit clears every node of the run. A load-only run's rows have status loaded_only. Only job nodes' run() takes an argument (load_only). A loaded job (PipelineJob.load) runs only with load_only=True. + job.run()/run_async() use the same state rule and plan (node_run.plan_run): a current node is + held (not run; its rows feed its successors, both orchestrators take held rows), sources + are read only for nodes that run, a failed end-of-run commit clears only the nodes that ran, + and the summary reports held nodes (NodeRunSummary.held). REPLAY replays every logged run + of its key (ITL-1098). - Empty data → ArrowTableSource raises ValueError("Table is empty"). - A side-effect job node delivers only when it runs (developer, November 13, 2026, ITL-1091): job.run(), node.run() (state-driven, ITL-1095: a node holding a current run delivers diff --git a/CLAUDE.md b/CLAUDE.md index 1ada669b9..b00c01c76 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -844,6 +844,11 @@ data. The rules: commit clears every node of the run. A load-only run's rows have status `loaded_only`. Only job nodes' `run()` takes an argument (`load_only`). A loaded job (`PipelineJob.load`) runs only with `load_only=True`. + `job.run()`/`run_async()` use the same state rule and plan (`node_run.plan_run`): a current node is + held (not run; its rows feed its successors, both orchestrators take held rows), sources + are read only for nodes that run, a failed end-of-run commit clears only the nodes that ran, + and the summary reports held nodes (`NodeRunSummary.held`). REPLAY replays every logged run + of its key (ITL-1098). - Empty data → `ArrowTableSource` raises `ValueError("Table is empty")`. - A side-effect job node delivers only when it runs (developer, November 13, 2026, ITL-1091): `job.run()`, `node.run()` (state-driven, ITL-1095: a node holding a current run delivers diff --git a/changelog.d/ITL-1095.md b/changelog.d/ITL-1095.md index 7675b59a9..ddb1c473c 100644 --- a/changelog.d/ITL-1095.md +++ b/changelog.d/ITL-1095.md @@ -5,6 +5,7 @@ - A loaded operator node no longer raises from `as_table()` or `iter_data()`: it is empty, like every node that has not run. Its stored view (`as_source()`, `get_all_records()`) raises instead when the job file does not record its run's content hash and the store holds nothing under the hash the loaded job computes (ITL-1043). - A job node's `run()` is state-driven and runs what the node depends on (developer, November 13, 2026; ITL-1095). A node that holds a completed run and is not stale runs nothing: `run()` returns the rows it holds. Otherwise every job node it depends on that holds no current run runs first, each once, as `job.run()` would run that part of the job, then the node; the run's writes are committed when it ends (before, `node.run()` read an upstream node's stored rows, or nothing, and left its last pipeline record and a `LOG` operator's output uncommitted). `run()` of a node whose function pod is a placeholder, or that was loaded without one, raises unless `load_only=True`; a `CACHE_ONLY` node's `run()` raises (its input is unavailable): read its stored results with `as_source()`. A job node's `run()` takes one keyword-only argument, `load_only`; every other stream's `run()` still takes none. - A side-effect job node follows the same rule: it is stale before a run, and while it holds a current run its `run()` (or a downstream node's `run()`) delivers nothing; to deliver again without `track_completion`, call `clear_cache()` on it first (`job.run()` still delivers as before). A load-only run passes its rows on and delivers nothing. +- `job.run()` and `job.run_async()` follow the same rule as `node.run()` (developer, November 14, 2026): a node that holds a current run is not run again, and its rows feed the nodes after it. Running an unchanged job a second time now computes nothing, reads and writes nothing in the store, does not read its sources and delivers no side effect, even without `track_completion` (before, every `job.run()` re-read every source and re-processed every node, and delivered every untracked side effect again). Nodes run when they have not run in this process, were cleared, are below a cleared node or a node that ran again, or below a source bound again or whose own modified time moved. A source whose data changes while its object stays the same (a file or table changed in place) is not seen: bind a new source object, or call `clear_cache()` on the first node below it, to read it again. The summary reports a node that was not run as held: "rowwise: 6 rows: held from the previous run" (`NodeRunSummary.held`). If the commit at the end of a run fails, the nodes it ran are cleared and the nodes it held keep their rows. - If the commit at the end of `job.run()` or `node.run()` fails, every function and operator node of the run is cleared (not run, stale) before the error propagates. ### Added diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 8703940cc..dc65dda8b 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -578,30 +578,47 @@ incomplete: a group formed after it may lack members, and the result stored for group by an earlier run may then be loaded for it. `run_async(load_only=True)` does the same on the asynchronous orchestrator. -### Running one node: `node.run()` +### Running again: what a run runs -`job.nodes[label].run()` runs that node as `job.run()` would, together with the job nodes -it depends on, and returns a view of its rows. What runs depends on each node's state -(developer, November 13, 2026): +`job.run()`, `await job.run_async()` and `job.nodes[label].run()` decide what to run from +each node's state (developer, November 13 and 14, 2026). `node.run()` runs that node +together with the job nodes it depends on and returns a view of its rows; `job.run()` does +the same for every node of the job. -| State of a node | What it shows | `run()` | `run(load_only=True)` | +| State of a node | What it shows | A run | A load-only run | |---|---|---|---| | Not run in this process (a fresh or loaded job), cleared, or stale | nothing: `head()` says "not run" (`not_computed`) | runs it | loads it | -| Filled by a load-only run | the rows it loaded: status `loaded_only`, "N loaded, M inputs without a stored result" | runs it, computing what it skipped | nothing: shows what it holds | -| Filled by a run (`job.run()` or `run()`) | that run's rows (`rows`) | nothing: shows what it holds | nothing: shows what it holds | - -A node is stale when anything it depends on changed after its run: an upstream node ran -again or was cleared, or a source was bound again (`job.bind(sources=...)`). Before a node -runs, every job node it depends on that is not current for that kind of run runs first, -each once, in order; the others feed it their rows. A load-only run's rows are never current -for a full run, nor are the rows of any node below them. `node.last_run` holds the counts, -as `job.last_run` does for `job.run()`, and the run's writes are committed when it ends. -Side effects above the node are delivered as `job.run()` delivers them; a node that holds a -current run is not run again, so to deliver a side effect again (with -`track_completion=False`), call `clear_cache()` on its node first. Staleness comes from -modified times only: a source whose rows change without its object changing (a table read -lazily, a file rewritten in place) is not seen. Build a new source object and bind it, or -clear the nodes, to run them again. +| Filled by a load-only run | the rows it loaded: status `loaded_only`, "N loaded, M inputs without a stored result" | runs it, computing what it skipped | holds it | +| Filled by a run | that run's rows (`rows`) | holds it | holds it | + +A node that is held is not run: its rows feed the nodes after it, and nothing is computed, +read from the store, written or delivered for it. So running an unchanged job again does +nothing, also on the asynchronous orchestrator, and the summary says so per node: +"rowwise: 6 rows: held from the previous run" (`summary.nodes["rowwise"].held`). A load-only +run's rows are never held by a full run, nor are the rows of any node below them. A source +is read only when a node that runs reads it. + +A node is stale, and runs, when: + +- it has not run in this process, its last run raised, or the commit at the end of its run + failed; +- `clear_cache()` was called on it, or on any job node above it; +- a job node above it ran again; +- its source was bound again (`job.bind(sources=...)`), or the bound source's own modified + time moved (a `CachedSource` that read new rows, a `PollingSource` that polled). + +Staleness comes from modified times only. A source whose rows change while its object stays +the same (a CSV or Delta file rewritten in place, a database table that grew, a list a +`DictSource` was built from and that was changed afterwards) is not seen: a run holds the +nodes below it. To read it again, build a new source object and bind it +(`job.bind(sources=...)`), or call `clear_cache()` on the first node below it. To deliver a +side effect again (with `track_completion=False`), call `clear_cache()` on its node first. + +`node.last_run` holds the counts of a `node.run()`, as `job.last_run` does for `job.run()`, +and a run's writes are committed when it ends; when everything is held there is nothing to +commit. If that commit fails, the nodes the run ran are cleared (they read as not run); the +nodes it held keep their rows. A held node fires no observer hook and cannot fail, so the +orchestrator's `error_policy` has nothing to apply to it. ### When results are committed @@ -666,6 +683,9 @@ keeps that output in memory and, with the default `cache_mode=OFF`, stores nothi `get_all_records()` returns `None` when no matching records exist; records from an earlier LOG run remain readable after switching to OFF. Calling `clear_cache()` on it discards the output, and it then reads as a zero-row table, as before the first run, until the job runs again. +An operator node attached in `CacheMode.REPLAY` does not compute: its run replays the rows +stored under its key by `CacheMode.LOG` runs, every logged run of that key, not only the +latest; before its run it shows nothing, like every node (a narrower replay is ITL-1098). ### When the store cannot be written diff --git a/src/orcapod/pipeline/async_orchestrator.py b/src/orcapod/pipeline/async_orchestrator.py index 6cc442198..a2c2cf197 100644 --- a/src/orcapod/pipeline/async_orchestrator.py +++ b/src/orcapod/pipeline/async_orchestrator.py @@ -31,6 +31,9 @@ ) if TYPE_CHECKING: + from collections.abc import Mapping + + from orcapod.channels import WritableChannel from orcapod.pipeline.dag import GraphProtocol from orcapod.protocols.core_protocols import DataProtocol, TagProtocol from orcapod.protocols.observability_protocols import ExecutionObserverProtocol @@ -76,6 +79,7 @@ def run( run_id: str | None = None, pipeline_uri: str = "", load_only: bool = False, + held_rows: Mapping[Any, list[tuple[TagProtocol, DataProtocol]]] | None = None, ) -> OrchestratorResult: """Synchronous entry point — runs the async pipeline to completion. @@ -120,6 +124,7 @@ def run( run_id=run_id, pipeline_uri=pipeline_uri, load_only=load_only, + held_rows=held_rows, ) ) @@ -141,6 +146,7 @@ def make_run() -> Coroutine[Any, Any, OrchestratorResult]: run_id=run_id, pipeline_uri=pipeline_uri, load_only=load_only, + held_rows=held_rows, ) return _run_on_own_thread(make_run, rows_in_flight) @@ -154,6 +160,7 @@ async def run_async( run_id: str | None = None, pipeline_uri: str = "", load_only: bool = False, + held_rows: Mapping[Any, list[tuple[TagProtocol, DataProtocol]]] | None = None, ) -> OrchestratorResult: """Async entry point for callers already inside an event loop. @@ -165,6 +172,9 @@ async def run_async( generated automatically. pipeline_uri: Opaque URI string identifying this pipeline run. Forwarded verbatim to ``observer.on_run_start``. + load_only: As for ``SyncPipelineOrchestrator.run``. + held_rows: As for ``SyncPipelineOrchestrator.run``: nodes not run, whose + rows feed their successors. Returns: OrchestratorResult with node outputs. @@ -176,6 +186,7 @@ async def run_async( run_id=run_id, pipeline_uri=pipeline_uri, load_only=load_only, + held_rows=held_rows, ) async def _run_async( @@ -187,6 +198,7 @@ async def _run_async( run_id: str | None = None, pipeline_uri: str = "", load_only: bool = False, + held_rows: Mapping[Any, list[tuple[TagProtocol, DataProtocol]]] | None = None, ) -> OrchestratorResult: """Core async logic: wire channels, launch tasks, collect results.""" from orcapod.pipeline.observer import NoOpObserver @@ -246,7 +258,10 @@ async def _run_async( collector = collectors[node] writer = _CollectingWriter(writer, collector) - if is_source_node(node): + if held_rows is not None and node in held_rows: + # Not run: the rows of its current run feed its successors. + _start_node(tg, node, _send_held(held_rows[node], writer)) + elif is_source_node(node): _start_node( tg, node, @@ -608,6 +623,18 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: return NoOpLogger() +async def _send_held( + rows: list[tuple[TagProtocol, DataProtocol]], + writer: WritableChannel[tuple[TagProtocol, DataProtocol]], +) -> None: + """Send the rows of a node the run holds rather than runs, then close.""" + try: + for row in rows: + await writer.send(row) + finally: + await writer.close() + + def _start_node( tg: asyncio.TaskGroup, node: Any, work: Coroutine[Any, Any, None] ) -> None: diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 91644b9a5..3f720a19d 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -28,7 +28,7 @@ ) from orcapod.pipeline.base import AbstractPipelineBase from orcapod.pipeline.dag import OrcaDAG -from orcapod.pipeline.node_run import clear_run_rows +from orcapod.pipeline.node_run import RunPlan, clear_run_rows, plan_run from orcapod.pipeline.run_summary import ( RunCountingObserver, RunSummary, @@ -851,6 +851,14 @@ def run( After a successful run, ``job.nodes`` returns the execution-ready nodes with populated database caches. + State-driven, as every job node's ``run()`` (developer, November 14, 2026; + ITL-1095): a node that holds a current run (completed in this process, not + stale, and for a full run not filled by a load-only run) is held, not run; its + rows feed the nodes after it, and nothing is computed, read, written or + delivered for it. A source is read only for the nodes that run. So a second run + of an unchanged job does nothing. To run a node again, call ``clear_cache()`` on + it (every node below it runs too), or bind a new source object. + ``job.run(orchestrator=AsyncPipelineOrchestrator())`` works the same in a script and inside a running event loop, such as a Jupyter kernel: there the run goes on a separate thread while the call waits, and blocks that loop @@ -930,18 +938,21 @@ def run( from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator prepared = self._prepare_run(commit_every_rows, load_only=load_only) + # The state rule (ITL-1095): a node that holds a current run is not run again. + plan = plan_run(prepared.exec_dag, load_only=load_only) counter = RunCountingObserver() effective_orchestrator = orchestrator or SyncPipelineOrchestrator() - with self._executing(prepared, commit_every_rows): + with self._executing(prepared, plan, commit_every_rows): _ = effective_orchestrator.run( - prepared.exec_dag, + plan.dag, observer=_with_counter(observer, counter), run_id=prepared.run_id, pipeline_uri=prepared.pipeline_uri, materialize_results=materialize_results, load_only=load_only, + held_rows=plan.held_rows, ) - summary = self._finish_run(prepared, counter, load_only=load_only) + summary = self._finish_run(prepared, counter, plan, load_only=load_only) if not summary.ok: warn_row_failures(summary, sys._getframe(1)) # pyright: ignore[reportPrivateUsage] # the caller's line is where the warning is shown return summary @@ -993,17 +1004,20 @@ async def run_async( "it." ) prepared = self._prepare_run(commit_every_rows, load_only=load_only) + # The state rule (ITL-1095): a node that holds a current run is not run again. + plan = plan_run(prepared.exec_dag, load_only=load_only) counter = RunCountingObserver() - with self._executing(prepared, commit_every_rows): + with self._executing(prepared, plan, commit_every_rows): _ = await orchestrator.run_async( - prepared.exec_dag, + plan.dag, observer=_with_counter(observer, counter), run_id=prepared.run_id, pipeline_uri=prepared.pipeline_uri, materialize_results=materialize_results, load_only=load_only, + held_rows=plan.held_rows, ) - summary = self._finish_run(prepared, counter, load_only=load_only) + summary = self._finish_run(prepared, counter, plan, load_only=load_only) if not summary.ok: warn_row_failures(summary, sys._getframe(1)) # pyright: ignore[reportPrivateUsage] # the caller's line is where the warning is shown return summary @@ -1187,7 +1201,7 @@ def _prepare_run( @contextlib.contextmanager def _executing( - self, prepared: "_PreparedRun", commit_every_rows: int + self, prepared: "_PreparedRun", plan: RunPlan, commit_every_rows: int ) -> Iterator[None]: """Hold every function node to the run's commit cadence while the orchestrator runs. @@ -1195,12 +1209,13 @@ def _executing( Args: prepared: The run, from ``_prepare_run``. + plan: The nodes the run runs (``plan_run``). commit_every_rows: As for ``run``. """ try: with contextlib.ExitStack() as batched: # Function nodes commit every commit_every_rows rows (ITL-919). - for node in prepared.exec_dag.nodes(): + for node in plan.to_run: if isinstance(node, FunctionJobNode): batched.enter_context( node._batched_commits(commit_every_rows) # pyright: ignore[reportPrivateUsage] # the job owns its nodes' commit cadence @@ -1216,14 +1231,19 @@ def _finish_run( self, prepared: "_PreparedRun", counter: RunCountingObserver, + plan: RunPlan, *, load_only: bool = False, ) -> RunSummary: """Commit the run's last writes, mark the job as run and keep its summary. + A failed commit clears the nodes the run ran (they hold no completed run); + the nodes it held keep the rows of their earlier run. + Args: prepared: The run, from ``_prepare_run``. counter: The observer that counted the run's rows. + plan: The nodes the run ran and held (``plan_run``). load_only: Whether the run loaded stored results only. Returns: @@ -1249,12 +1269,12 @@ def _finish_run( # A stored table could not be read: report the read failure itself. exc.add_note(pending_note) # A run whose commit failed did not complete: no node holds its rows. - clear_run_rows(prepared.exec_dag.nodes()) + clear_run_rows(plan.to_run) raise except Exception as exc: flush_error = StoreWriteError(pipeline_db.base_path, flush=True) flush_error.add_note(pending_note) - clear_run_rows(prepared.exec_dag.nodes()) + clear_run_rows(plan.to_run) raise flush_error from exc # Mutate self in place. @@ -1263,7 +1283,7 @@ def _finish_run( self._run_id = prepared.run_id self._last_run = RunSummary( "/".join(self._name), - run_node_summaries(prepared.exec_dag, counter), + run_node_summaries(prepared.exec_dag, counter, held=plan.held_counts), load_only=load_only, ) return self._last_run diff --git a/src/orcapod/pipeline/node_run.py b/src/orcapod/pipeline/node_run.py index a7d4b1fd9..946b24b1d 100644 --- a/src/orcapod/pipeline/node_run.py +++ b/src/orcapod/pipeline/node_run.py @@ -14,7 +14,8 @@ from __future__ import annotations -from collections.abc import Iterable +from collections.abc import Iterable, Sequence +from dataclasses import dataclass from typing import TYPE_CHECKING, Any from orcapod.errors import StoreError, StoreWriteError @@ -85,89 +86,172 @@ def clear_run_rows(nodes: Iterable[Any]) -> None: node.clear_cache() # pyright: ignore[reportAttributeAccessIssue] # every RunHoldingNode is a job node with clear_cache -def run_with_dependencies(target: Any, *, load_only: bool) -> RunSummary: - """Run ``target`` and every job node it depends on that holds no current run. +@dataclass(frozen=True) +class RunPlan: + """What a run does with each node of its graph (ITL-1095). + + Attributes: + dag: The nodes the run touches: those it runs, and the predecessors that feed + them (a source slot is read; any other one is held). + held_rows: The rows of each held node of ``dag``: the rows of its current run, + or of a stream outside the job; they feed the nodes after it as if it had + just run. + to_run: The nodes the run runs, in dependency order. + held_counts: Every node of the whole graph that holds a current run, with its + row count, for the summary. + """ - The dependencies are found through ``upstreams``. A dependency that holds a current - run (``holds_current_run``) is not run: its rows feed the nodes after it. A stream - outside the job is read. Every other node runs once, in dependency order, on the - ``SyncPipelineOrchestrator`` (error policy ``"continue"``), as ``job.run()`` would - run that sub-graph; with ``load_only`` as ``job.run(load_only=True)`` would. The run - ends with a commit of every store it wrote to, also, best-effort, when it fails. + dag: Any + held_rows: dict[Any, list[Any]] + to_run: list[Any] + held_counts: dict[Any, int] + + +def plan_run(graph: Any, *, load_only: bool, outside: Sequence[Any] = ()) -> RunPlan: + """Choose the nodes a run runs: the state rule of every job node's run (ITL-1095). + + A node that holds a current run (``holds_current_run``) is not run; a source slot + is read only when a node that runs reads it; a stream outside the job + (``outside``) is read for the nodes it feeds; every other node runs. The same plan + serves ``job.run()`` (its whole graph) and ``node.run()`` (the node and what it + depends on). Args: - target: The job node whose ``run()`` this is. - load_only: Whether to load stored results only. + graph: The run's node graph (``topological_sort``, ``predecessors``). + load_only: Whether the run is load-only (a node filled by a load-only run is + current only for one). + outside: Streams of ``graph`` that are not job nodes. Returns: - The run's summary, one entry per function node that ran. - - Raises: - StoreWriteError: If committing the run's writes failed. + The plan. """ from orcapod.core.nodes.source_node import SourceJobNode from orcapod.core.nodes.staleness import RunHoldingNode from orcapod.pipeline.dag import OrcaDAG - from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator - dag: OrcaDAG[Any] = OrcaDAG() - held: dict[Any, list[Any]] = {} - seen: list[Any] = [] + order = list(graph.topological_sort()) + + def is_outside(node: Any) -> bool: + return any(node is other for other in outside) def is_current(node: Any) -> bool: - """Whether ``node`` holds a run this run does not redo.""" return isinstance(node, RunHoldingNode) and node.holds_current_run( load_only=load_only ) def is_source(node: Any) -> bool: - """Whether ``node`` is a job's source slot: read by the run, never held.""" return isinstance(node, SourceJobNode) + current = [node for node in order if is_current(node)] + to_run = [ + node + for node in order + if not is_source(node) + and not is_outside(node) + and not any(node is c for c in current) + ] + included: list[Any] = [] + for node in order: + feeds_a_run = any( + any(successor is r for r in to_run) for successor in graph.successors(node) + ) + if any(node is r for r in to_run) or feeds_a_run: + included.append(node) + dag: OrcaDAG[Any] = OrcaDAG() + for node in included: + dag.add_node(node) + for node in to_run: + for upstream in graph.predecessors(node): + dag.add_edge(upstream, node) + held_rows = { + node: list(node.iter_data()) + for node in included + if not any(node is r for r in to_run) and not is_source(node) + } + held_counts = {node: sum(1 for _ in node.iter_data()) for node in current} + return RunPlan(dag, held_rows, to_run, held_counts) + + +def commit_run(stores: Sequence[ArrowDatabaseProtocol], ran: Sequence[Any]) -> None: + """Commit a run's writes at its end; a failed commit clears the nodes that ran. + + A run whose commit failed did not complete, so no node it ran holds its rows + (ITL-1095); nodes it held keep the rows of their earlier run. + + Raises: + StoreError: The commit's own store error. + StoreWriteError: If committing failed for another reason. + """ + for store in stores: + try: + store.flush() + except StoreError: + clear_run_rows(ran) + raise + except Exception as exc: + clear_run_rows(ran) + raise StoreWriteError(store.base_path, flush=True) from exc + + +def run_with_dependencies(target: Any, *, load_only: bool) -> RunSummary: + """Run ``target`` and every job node it depends on that holds no current run. + + The dependencies are found through ``upstreams``; the plan (``plan_run``) is the + one ``job.run()`` makes over the job: a dependency that holds a current run is not + run and its rows feed the nodes after it, a stream outside the job is read, every + other node runs once, in dependency order, on the ``SyncPipelineOrchestrator`` + (error policy ``"continue"``). The run ends with a commit of every store it wrote + to, also, best-effort, when it fails. + + Args: + target: The job node whose ``run()`` this is. + load_only: Whether to load stored results only. + + Returns: + The run's summary, one entry per function node of the graph. + + Raises: + StoreWriteError: If committing the run's writes failed. + """ + from orcapod.core.nodes.source_node import SourceJobNode + from orcapod.pipeline.dag import OrcaDAG + from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator + + graph: OrcaDAG[Any] = OrcaDAG() + seen: list[Any] = [] + outside: list[Any] = [] + def visit(node: Any) -> None: - """Add ``node`` and, unless it is held or a source, what it depends on.""" + """Add ``node`` and what it depends on, up to the sources of the job.""" if any(node is other for other in seen): return seen.append(node) - dag.add_node(node) - if node is not target and is_current(node): - held[node] = list(node.iter_data()) - return - if is_source(node): + graph.add_node(node) + if isinstance(node, SourceJobNode): return if not _is_job_node(node): - # A stream outside the job: its rows, read as the node's run reads them. - held[node] = list(node.iter_data()) + outside.append(node) return for upstream in node.upstreams: visit(upstream) - dag.add_edge(upstream, node) + graph.add_edge(upstream, node) visit(target) - run_nodes = [node for node in seen if not any(node is h for h in held)] - stores = _stores(run_nodes) + plan = plan_run(graph, load_only=load_only, outside=outside) + stores = _stores(plan.to_run) counter = RunCountingObserver() try: _ = SyncPipelineOrchestrator().run( - dag, observer=counter, load_only=load_only, held_rows=held + plan.dag, observer=counter, load_only=load_only, held_rows=plan.held_rows ) except Exception as run_error: # Commit the writes that succeeded before the failure, as job.run() does. flush_best_effort(run_error, *stores) raise - for store in stores: - try: - store.flush() - except StoreError: - # A run whose commit failed did not complete: no node holds its rows. - clear_run_rows(run_nodes) - raise - except Exception as exc: - clear_run_rows(run_nodes) - raise StoreWriteError(store.base_path, flush=True) from exc + commit_run(stores, plan.to_run) return RunSummary( _job_name(target), - run_node_summaries(dag, counter, skip=list(held)), + run_node_summaries(graph, counter, held=plan.held_counts), load_only=load_only, target=target.label, ) diff --git a/src/orcapod/pipeline/run_summary.py b/src/orcapod/pipeline/run_summary.py index ee48b49f3..d003dde3d 100644 --- a/src/orcapod/pipeline/run_summary.py +++ b/src/orcapod/pipeline/run_summary.py @@ -18,7 +18,7 @@ from orcapod.pipeline.observer import NoOpLogger if TYPE_CHECKING: - from collections.abc import Collection, Mapping, Sequence + from collections.abc import Mapping, Sequence from types import FrameType from orcapod.protocols.core_protocols import DataProtocol, TagProtocol @@ -43,6 +43,8 @@ class NodeRunSummary: inputs_incomplete: The labels of the function nodes upstream of this one that skipped inputs in a load-only run, in run order: this node's inputs lack the rows those results would have given (a group may be partial). + held: The rows of a node the run did not run because it held a current run + from its previous run (ITL-1095): it computed, read and wrote nothing. """ label: str @@ -52,11 +54,12 @@ class NodeRunSummary: first_error: BaseException | None not_stored: int = 0 inputs_incomplete: tuple[str, ...] = () + held: int = 0 @property def rows(self) -> int: - """All rows the node received: computed, reused, failed and not stored.""" - return self.computed + self.reused + self.failed + self.not_stored + """All rows of the node: computed, reused, failed, not stored and held.""" + return self.computed + self.reused + self.failed + self.not_stored + self.held class RunSummary: @@ -239,6 +242,8 @@ def _load_only_html(self) -> str: def _counts(node: NodeRunSummary, load_only: bool) -> str: """A node's counts as text: what a full run did, or what a load-only run loaded.""" + if node.held and not (node.computed or node.reused or node.failed): + return "held from the previous run" if not load_only: return ( f"{node.computed} computed, {node.reused} from store, {node.failed} failed" @@ -394,7 +399,7 @@ def create_data_logger(self, tag: TagProtocol, data: DataProtocol) -> Any: def run_node_summaries( - dag: Any, counter: RunCountingObserver, skip: Collection[Any] = () + dag: Any, counter: RunCountingObserver, held: Mapping[Any, int] | None = None ) -> list[NodeRunSummary]: """One summary per function node of a finished run, in run order. @@ -405,9 +410,9 @@ def run_node_summaries( Args: dag: The run's node graph (``topological_sort``, ``predecessors``). counter: The observer that counted the run's rows. - skip: Nodes of ``dag`` the run did not run (their rows were held): they get no - summary, but inputs they skipped in an earlier load-only run still make the - nodes below them incomplete. + held: Nodes of ``dag`` the run did not run because they hold a current run, + with their row counts: reported as held (ITL-1095). Inputs they skipped in + an earlier load-only run still make the nodes below them incomplete. Returns: The summaries of the run's function nodes, in run order. @@ -430,14 +435,25 @@ def run_node_summaries( if getattr(upstream, "last_run_not_stored", 0): above.add(id(upstream)) skipping_above[id(node)] = above - if any(node is skipped for skipped in skip): + # A stream outside the job (in a node.run() graph) is no node of the run. + if getattr(node, "node_type", None) is None: continue if not (is_function_node(node) or is_side_effect_function_node(node)): continue + held_rows = next( + (count for other, count in (held or {}).items() if other is node), None + ) labels = tuple( order[index].label for index in sorted(position[key] for key in above) ) label: str = getattr(node, "label") # every graph node is labelled + if held_rows is not None: + summaries.append( + NodeRunSummary( + label, 0, 0, 0, None, inputs_incomplete=labels, held=held_rows + ) + ) + continue summaries.append( replace( counter.node_summary(label), diff --git a/tests/test_core/side_effect_pod/test_side_effect_job_node_reads.py b/tests/test_core/side_effect_pod/test_side_effect_job_node_reads.py index cb272bca9..f03747ac3 100644 --- a/tests/test_core/side_effect_pod/test_side_effect_job_node_reads.py +++ b/tests/test_core/side_effect_pod/test_side_effect_job_node_reads.py @@ -199,6 +199,11 @@ def test_node_run_without_track_completion_delivers_again_only_after_clear_cache view = job.nodes["delivered"].run() assert len(sent) == 2 * ROWS assert len(view) == ROWS + # job.run() follows the same rule: the node holds a current run, so it delivers + # nothing; cleared, it delivers every row again. + _run(job, orchestrator) + assert len(sent) == 2 * ROWS + job.nodes["delivered"].clear_cache() _run(job, orchestrator) assert len(sent) == 3 * ROWS # The reads after either kind of run deliver nothing. @@ -224,6 +229,8 @@ def test_an_async_run_holds_exactly_its_own_rows(track_completion: bool) -> None """After two async runs, the node holds the rows of the last run only.""" job, sent = _job(track_completion=track_completion) _run(job, "async") + # A node holding a current run is not run again (ITL-1095): clear it first. + job.nodes["delivered"].clear_cache() _run(job, "async") node = job.nodes["delivered"] assert len(node.as_table()) == ROWS diff --git a/tests/test_pipeline/test_hash_step_combined.py b/tests/test_pipeline/test_hash_step_combined.py index 7459a4bad..c8880a3b6 100644 --- a/tests/test_pipeline/test_hash_step_combined.py +++ b/tests/test_pipeline/test_hash_step_combined.py @@ -226,7 +226,10 @@ def test_a_table_written_before_v03_that_appears_between_runs_is_refused(self): # A log of the same pod at the layout Orcapod v0.2 wrote, under another hash. old_path = _add_table_written_before_v03(store, "tdb_v2", "tdb_v1") - # The same job object runs again: the check is not remembered from the first run. + # The same job object runs the sink again (a node holding a current run is not + # run, ITL-1095, so it is cleared first): the check is not remembered from the + # first run. + job.nodes["sink"].clear_cache() with pytest.raises(SideEffectLogDataContextError) as info: _ = job.run() assert old_path in info.value.record_paths @@ -239,11 +242,13 @@ def test_a_function_pod_with_ctx_arg_checks_before_every_run_too(self): _ = job.run() assert sorted(delivered) == [1, 2] delivered.clear() + job.nodes["notify"].clear_cache() # a current node is not run again (ITL-1095) _ = job.run() # nothing new in the store: runs, and delivers nothing again assert delivered == [] old_path = _add_table_written_before_v03(store, "pdb_v2", "pdb_v1") + job.nodes["notify"].clear_cache() with pytest.raises(SideEffectLogDataContextError) as info: _ = job.run() assert old_path in info.value.record_paths diff --git a/tests/test_pipeline/test_job_node_rows_after_run.py b/tests/test_pipeline/test_job_node_rows_after_run.py index c10cdea19..e5337186b 100644 --- a/tests/test_pipeline/test_job_node_rows_after_run.py +++ b/tests/test_pipeline/test_job_node_rows_after_run.py @@ -465,6 +465,8 @@ def on_node_end(self, node_label: str, node_hash: str) -> None: job.run(orchestrator=_orchestrator(kind)) node = job.nodes["node"] assert _values(node, lambda r: r["k"]) == [0, 1, 2] + # A node holding a current run is not run again (ITL-1095): clear it first. + node.clear_cache() # The async orchestrator raises its task group's error around it. with pytest.raises((RuntimeError, ExceptionGroup)): job.run(orchestrator=_orchestrator(kind), observer=FailsAtNodeEnd()) diff --git a/tests/test_pipeline/test_sync_orchestrator.py b/tests/test_pipeline/test_sync_orchestrator.py index aaa4c9b68..7fc3b67e4 100644 --- a/tests/test_pipeline/test_sync_orchestrator.py +++ b/tests/test_pipeline/test_sync_orchestrator.py @@ -560,6 +560,9 @@ def create_data_logger(self, tag, data, **kwargs): def contextualize(self, *identity_path): return self + # A node holding a current run is not run again (ITL-1095): clear it, so its + # second run reads its result from the store. + job.nodes["doubler"].clear_cache() job.run(observer=Obs2()) assert events2 == [True] From 3cd6bef52a1f2f0501abb12f179e280f973fc41c Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 02:37:31 -0800 Subject: [PATCH 31/63] fix(nodes): review fixes for the current-run rule and load-only runs (ITL-1095) CodeRabbit review of c1a97dea, plan approved by the coordinator: - a load-only run never flushes (job.run prepare/failure/finish, the node cascade, a node or side-effect node run alone): pending writes of others stay pending; - a run that raises (an observer at a node's end, a failed store write or commit, a failed or cancelled output close) leaves no node it ran holding rows; held nodes keep theirs; a failed side-effect run makes the nodes below it stale; - a standalone LOG operator run commits its write; a full standalone run after a load-only run computes and stores; an operator filled by a load-only run shows LOADED_ONLY; an operator rebuilt without its operator keeps its saved columns; - a source slot bound to a stream with no modified time has none (stale below); - staleness and currency checks visit each ancestor once per traversal; - a held dependency keeps its incomplete-input ancestry in the summary (test); - a run with every node held says so; a loaded job that lost a node names ITL-1094 instead of failing in the orchestrator; - staleness tests start from a run (nitpick). Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1095.md | 2 +- docs/concepts/function-pods.md | 7 +- src/orcapod/core/nodes/function_node.py | 18 +- src/orcapod/core/nodes/operator_node.py | 142 +++-- src/orcapod/core/nodes/source_node.py | 19 +- src/orcapod/core/nodes/staleness.py | 60 ++- src/orcapod/core/streams/view.py | 6 + src/orcapod/pipeline/job.py | 45 +- src/orcapod/pipeline/node_run.py | 16 +- src/orcapod/pipeline/run_summary.py | 6 + src/orcapod/side_effects.py | 37 +- .../test_function_pod_node_stream.py | 10 +- .../test_current_run_review_fixes.py | 490 ++++++++++++++++++ 13 files changed, 763 insertions(+), 95 deletions(-) create mode 100644 tests/test_pipeline/test_current_run_review_fixes.py diff --git a/changelog.d/ITL-1095.md b/changelog.d/ITL-1095.md index ddb1c473c..4629ac44b 100644 --- a/changelog.d/ITL-1095.md +++ b/changelog.d/ITL-1095.md @@ -5,7 +5,7 @@ - A loaded operator node no longer raises from `as_table()` or `iter_data()`: it is empty, like every node that has not run. Its stored view (`as_source()`, `get_all_records()`) raises instead when the job file does not record its run's content hash and the store holds nothing under the hash the loaded job computes (ITL-1043). - A job node's `run()` is state-driven and runs what the node depends on (developer, November 13, 2026; ITL-1095). A node that holds a completed run and is not stale runs nothing: `run()` returns the rows it holds. Otherwise every job node it depends on that holds no current run runs first, each once, as `job.run()` would run that part of the job, then the node; the run's writes are committed when it ends (before, `node.run()` read an upstream node's stored rows, or nothing, and left its last pipeline record and a `LOG` operator's output uncommitted). `run()` of a node whose function pod is a placeholder, or that was loaded without one, raises unless `load_only=True`; a `CACHE_ONLY` node's `run()` raises (its input is unavailable): read its stored results with `as_source()`. A job node's `run()` takes one keyword-only argument, `load_only`; every other stream's `run()` still takes none. - A side-effect job node follows the same rule: it is stale before a run, and while it holds a current run its `run()` (or a downstream node's `run()`) delivers nothing; to deliver again without `track_completion`, call `clear_cache()` on it first (`job.run()` still delivers as before). A load-only run passes its rows on and delivers nothing. -- `job.run()` and `job.run_async()` follow the same rule as `node.run()` (developer, November 14, 2026): a node that holds a current run is not run again, and its rows feed the nodes after it. Running an unchanged job a second time now computes nothing, reads and writes nothing in the store, does not read its sources and delivers no side effect, even without `track_completion` (before, every `job.run()` re-read every source and re-processed every node, and delivered every untracked side effect again). Nodes run when they have not run in this process, were cleared, are below a cleared node or a node that ran again, or below a source bound again or whose own modified time moved. A source whose data changes while its object stays the same (a file or table changed in place) is not seen: bind a new source object, or call `clear_cache()` on the first node below it, to read it again. The summary reports a node that was not run as held: "rowwise: 6 rows: held from the previous run" (`NodeRunSummary.held`). If the commit at the end of a run fails, the nodes it ran are cleared and the nodes it held keep their rows. +- `job.run()` and `job.run_async()` follow the same rule as `node.run()` (developer, November 14, 2026): a node that holds a current run is not run again, and its rows feed the nodes after it. Running an unchanged job a second time now computes nothing, reads and writes nothing in the store, does not read its sources and delivers no side effect, even without `track_completion` (before, every `job.run()` re-read every source and re-processed every node, and delivered every untracked side effect again). Nodes run when they have not run in this process, were cleared, are below a cleared node or a node that ran again, or below a source bound again or whose own modified time moved. After this change a re-run does not notice new data in a source whose object is unchanged (a file or table changed in place, a polling source with new rows): bind a new source object, or call `clear_cache()` on the first node below it, to read it again (a start-of-run check for new data is ITL-1102). The summary reports a node that was not run as held: "rowwise: 6 rows: held from the previous run" (`NodeRunSummary.held`). If the commit at the end of a run fails, the nodes it ran are cleared and the nodes it held keep their rows. - If the commit at the end of `job.run()` or `node.run()` fails, every function and operator node of the run is cleared (not run, stale) before the error propagates. ### Added diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index dc65dda8b..aa183b589 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -609,9 +609,10 @@ A node is stale, and runs, when: Staleness comes from modified times only. A source whose rows change while its object stays the same (a CSV or Delta file rewritten in place, a database table that grew, a list a -`DictSource` was built from and that was changed afterwards) is not seen: a run holds the -nodes below it. To read it again, build a new source object and bind it -(`job.bind(sources=...)`), or call `clear_cache()` on the first node below it. To deliver a +`DictSource` was built from and that was changed afterwards, a polling source with new +rows) is not seen: a run holds the nodes below it. To read it again, build a new source +object and bind it (`job.bind(sources=...)`), or call `clear_cache()` on the first node +below it. (A start-of-run check for new data is planned, ITL-1102.) To deliver a side effect again (with `track_completion=False`), call `clear_cache()` on its node first. `node.last_run` holds the counts of a `node.run()`, as `job.last_run` does for `job.run()`, diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 8a3e93ed3..74cad51f2 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -1695,13 +1695,13 @@ def _execute_load_only( loaded_ids.append(entry_id) if entry[1] is not None: output.append((entry[0], entry[1])) + self._set_run_rows(loaded_ids, load_only=True, not_stored=not_stored) + ctx_obs.on_node_end(node_label, node_hash) + self._update_modified_time() except BaseException: - # A run that raised defines no rows. + # A run that raised (an observer at the node's end included) defines no rows. self.clear_cache() raise - self._set_run_rows(loaded_ids, load_only=True, not_stored=not_stored) - ctx_obs.on_node_end(node_label, node_hash) - self._update_modified_time() return output async def _async_execute_load_only( @@ -3515,6 +3515,7 @@ def _run_alone(self, *, load_only: bool) -> Any: """ from dataclasses import replace + from orcapod.pipeline.node_run import commit_run from orcapod.pipeline.run_summary import RunCountingObserver, RunSummary from orcapod.utils.store_errors import flush_best_effort @@ -3534,10 +3535,13 @@ def _run_alone(self, *, load_only: bool) -> Any: try: _ = self.execute(self._input_stream, observer=counter, load_only=load_only) except Exception as run_error: - flush_best_effort(run_error, *stores) + if not load_only: + # Commit what succeeded; a load-only run writes nothing (ITL-1095). + flush_best_effort(run_error, *stores) + self.clear_cache() raise - for store in stores: - store.flush() + if not load_only: + commit_run(stores, [self]) node_summary = replace( counter.node_summary(self.label), not_stored=self.last_run_not_stored ) diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 7a89d5588..10a55babe 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -17,6 +17,7 @@ from __future__ import annotations import asyncio +import dataclasses import sys import logging from abc import abstractmethod @@ -34,6 +35,7 @@ ) from orcapod.core.streams.view import ( JOB_NODE_NOT_RUN_NOTE, + LOADED_ONLY_OPERATOR_NOTE, STALE_NOTE, BLUEPRINT_NOTE, ViewStatus, @@ -1402,43 +1404,48 @@ def execute( obs = observer if observer is not None else NoOpObserver() ctx_obs = obs.contextualize(*self.node_identity_path) - ctx_obs.on_node_start(node_label, node_hash) + try: + ctx_obs.on_node_start(node_label, node_hash) - self._run_load_only = load_only - # Check REPLAY cache first - cached_output = self.get_cached_output() - if cached_output is not None: - output = list(cached_output.iter_data()) - ctx_obs.on_node_end(node_label, node_hash) - return output + self._run_load_only = load_only + # Check REPLAY cache first + cached_output = self.get_cached_output() + if cached_output is not None: + output = list(cached_output.iter_data()) + ctx_obs.on_node_end(node_label, node_hash) + return output - # Compute - result_stream = self._operator.process(*input_streams) + # Compute + result_stream = self._operator.process(*input_streams) - # Materialize - output = list(result_stream.iter_data()) + # Materialize + output = list(result_stream.iter_data()) - # Cache - if output: - self._cached_output_stream = StaticOutputOperatorPod._materialize_to_stream( - output - ) - else: - self._cached_output_stream = result_stream + # Cache + if output: + self._cached_output_stream = ( + StaticOutputOperatorPod._materialize_to_stream(output) + ) + else: + self._cached_output_stream = result_stream - self._update_modified_time() + self._update_modified_time() - # Persist to DB only in LOG mode, and never in a load-only run. - if ( - not load_only - and self._pipeline_database is not None - and self._cache_mode == CacheMode.LOG - and self._cached_output_stream is not None - ): - self._store_output_stream(self._cached_output_stream) + # Persist to DB only in LOG mode, and never in a load-only run. + if ( + not load_only + and self._pipeline_database is not None + and self._cache_mode == CacheMode.LOG + and self._cached_output_stream is not None + ): + self._store_output_stream(self._cached_output_stream) - ctx_obs.on_node_end(node_label, node_hash) - return output + ctx_obs.on_node_end(node_label, node_hash) + return output + except BaseException: + # A run that raised (its store write or an observer included) defines no rows. + self.clear_cache() + raise def _compute_and_store(self, *, store: bool = True) -> None: """Compute operator output, optionally store in DB. @@ -1540,24 +1547,36 @@ def run(self, *, load_only: bool = False) -> StreamView: return self._run_view(computed=False) if self._loaded: raise self._cannot_compute_error() - if self.is_stale: + if self.is_stale or not self._holds_run(load_only=load_only): + # Stale, or a load-only run's output asked for by a full run: run again. self.clear_cache() computed = False if self._cached_output_stream is None: - if self._pipeline_database is not None: - if self._cache_mode == CacheMode.REPLAY: - self._replay_from_cache() + try: + if self._pipeline_database is not None: + if self._cache_mode == CacheMode.REPLAY: + self._replay_from_cache() + else: + self._compute_and_store(store=not load_only) + computed = True + if self._cache_mode == CacheMode.LOG and not load_only: + # Committed before run() returns, as at the end of a job + # run. + from orcapod.pipeline.node_run import commit_run + + commit_run([self._pipeline_database], [self]) + self._run_load_only = load_only else: - self._compute_and_store(store=not load_only) + self._cached_output_stream = self._operator.process( + *self._input_streams, + ) + self._update_modified_time() computed = True - self._run_load_only = load_only - else: - self._cached_output_stream = self._operator.process( - *self._input_streams, - ) - self._update_modified_time() - computed = True + except BaseException: + # A run that raised defines no rows. + self.clear_cache() + raise return self._run_view(computed=computed) def _run_view(self, *, computed: bool) -> StreamView: @@ -1580,7 +1599,12 @@ def _run_view(self, *, computed: bool) -> StreamView: # computation, plain rows when replayed from the store or kept by execute(). kept = self._cached_output_stream origin = "computed" if kept is None else head_of(kept.head(1), self).origin - return self._view_of_rows(table, origin) + view = self._view_of_rows(table, origin) + if self._run_load_only: + return dataclasses.replace( + view, status=ViewStatus.LOADED_ONLY, note=LOADED_ONLY_OPERATOR_NOTE + ) + return view # Whether the output the node holds came from a load-only run (ITL-1095): its # input may lack rows, so a full run does not take it as complete. @@ -1633,9 +1657,18 @@ def as_table( current = self._current_output() if current is not None: return current.as_table(columns=columns, all_info=all_info) - # Requires a live operator for schema derivation (pre-existing limitation). if self._operator is None: - return pa.table({}) + # A node rebuilt without its operator: the columns of its saved schema. + tag_schema, data_schema = self.output_schema() + empty = pa.Table.from_pylist( + [], + schema=self.data_context.type_converter.python_schema_to_arrow_schema( + tag_schema + data_schema + ), + ) + return ArrowTableStream( + empty, tag_columns=tuple(tag_schema), data_context=self.data_context + ).as_table(columns=columns, all_info=all_info) empty_stream = ArrowTableStream( self._make_empty_table(), tag_columns=self.keys()[0], @@ -1668,7 +1701,13 @@ def head( return self._head_without_rows( config, ViewStatus.NOT_COMPUTED, note=STALE_NOTE ) - return head_of(self._cached_output_stream.head(n, columns=config), self) + head = head_of(self._cached_output_stream.head(n, columns=config), self) + if self._run_load_only: + # Output of a load-only run: its input may lack rows (ITL-1095). + return dataclasses.replace( + head, status=ViewStatus.LOADED_ONLY, note=LOADED_ONLY_OPERATOR_NOTE + ) + return head # ------------------------------------------------------------------ # DB retrieval @@ -1917,5 +1956,14 @@ async def forward() -> None: self._update_modified_time() ctx_obs.on_node_end(node_label, node_hash) + except BaseException: + # A run that raised or was cancelled defines no rows. + self.clear_cache() + raise finally: - await output.close() + try: + await output.close() + except BaseException: + # A close that raises or is cancelled fails the run: no rows. + self.clear_cache() + raise diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 07537c705..96353607b 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -628,17 +628,18 @@ def last_modified(self) -> datetime | None: The later of the time the slot was bound and the bound source's own ``last_modified`` (a ``CachedSource`` or ``PollingSource`` moves it when it - reads new rows). A bound source that does not know its modification time - leaves the binding time. A source whose rows change without its modified time - moving (a table read lazily, a file rewritten in place) is not seen: build a - new source object and bind it, or clear the dependent nodes. + reads new rows). A bound source that does not know its modification time makes + the slot's unknown too (``None``), so every node below it is stale. A source + whose rows change without its modified time moving (a table read lazily, a file + rewritten in place) is not seen: build a new source object and bind it, or clear + the dependent nodes. """ own = self._modified_time - bound = ( - self._bound_source.last_modified if self._bound_source is not None else None - ) - if own is None or bound is None: - return own if bound is None else bound + if self._bound_source is None: + return own + bound = self._bound_source.last_modified + if bound is None or own is None: + return bound return max(own, bound) # ------------------------------------------------------------------ diff --git a/src/orcapod/core/nodes/staleness.py b/src/orcapod/core/nodes/staleness.py index fc461a009..8ae8162a8 100644 --- a/src/orcapod/core/nodes/staleness.py +++ b/src/orcapod/core/nodes/staleness.py @@ -40,10 +40,15 @@ def is_stale(self) -> bool: True before the node's first run in this process and after ``clear_cache()`` (no modified time), when an input or its producer was modified after the node's run (``StreamBase.is_stale``), or when any job node it depends on is - stale (ITL-1095). A stale node reads empty until it runs again. + stale (ITL-1095). A stale node reads empty until it runs again. Each node of + the ancestry is checked once, however many paths lead to it. """ + return _stale(self, {}) + + def _own_staleness(self) -> bool: + """``StreamBase.is_stale`` of this node alone: its inputs and producer.""" own: bool = super().is_stale # pyright: ignore[reportAttributeAccessIssue] # the stream base after this mixin in the MRO defines it - return own or upstream_job_node_stale(self.upstreams) # pyright: ignore[reportAttributeAccessIssue] # every node class this is mixed into has upstreams + return own def holds_current_run(self, *, load_only: bool = False) -> bool: """Whether the node holds the rows of a run that ``run()`` would not redo. @@ -53,7 +58,8 @@ def holds_current_run(self, *, load_only: bool = False) -> bool: 2026, ITL-1095). A node filled by a load-only run is current only for another load-only run: a full run computes the inputs it skipped. So is every node whose rows came from such a node: a node is current only if every job node - above it is current for the same kind of run. + above it is current for the same kind of run. Each node of the ancestry is + checked once. Args: load_only: Whether the question is for a load-only run. @@ -61,20 +67,49 @@ def holds_current_run(self, *, load_only: bool = False) -> bool: Returns: ``True`` if the node holds a current run for such a run. """ - if not self._holds_run(load_only=load_only) or self.is_stale: - return False - upstreams: tuple[Any, ...] = getattr(self, "upstreams", ()) - return all( - upstream.holds_current_run(load_only=load_only) - for upstream in upstreams - if isinstance(upstream, RunHoldingNode) - ) + return _current(self, load_only, {}, {}) def _holds_run(self, *, load_only: bool) -> bool: """Whether the node holds the rows of a completed run good for such a run.""" raise NotImplementedError +def _job_upstreams(node: Any) -> list[RunHoldingNode]: + """The direct upstreams of ``node`` that are job nodes holding run rows.""" + upstreams: tuple[Any, ...] = getattr(node, "upstreams", ()) + return [upstream for upstream in upstreams if isinstance(upstream, RunHoldingNode)] + + +def _stale(node: RunHoldingNode, memo: dict[int, bool]) -> bool: + """Whether ``node`` is stale; ``memo`` keeps each node's answer for one traversal.""" + key = id(node) + if key not in memo: + memo[key] = node._own_staleness() or any( # pyright: ignore[reportPrivateUsage] # the mixin's own helper + _stale(upstream, memo) for upstream in _job_upstreams(node) + ) + return memo[key] + + +def _current( + node: RunHoldingNode, + load_only: bool, + memo: dict[int, bool], + stale_memo: dict[int, bool], +) -> bool: + """Whether ``node`` holds a current run; memoized for one traversal.""" + key = id(node) + if key not in memo: + memo[key] = ( + node._holds_run(load_only=load_only) # pyright: ignore[reportPrivateUsage] # the mixin's own hook + and not _stale(node, stale_memo) + and all( + _current(upstream, load_only, memo, stale_memo) + for upstream in _job_upstreams(node) + ) + ) + return memo[key] + + def upstream_job_node_stale(upstreams: Iterable[Any]) -> bool: """Return whether any job node in ``upstreams`` that holds run rows is stale. @@ -87,7 +122,8 @@ def upstream_job_node_stale(upstreams: Iterable[Any]) -> bool: Returns: ``True`` if any upstream ``RunHoldingNode`` is stale. """ + memo: dict[int, bool] = {} return any( - isinstance(upstream, RunHoldingNode) and upstream.is_stale + isinstance(upstream, RunHoldingNode) and _stale(upstream, memo) for upstream in upstreams ) diff --git a/src/orcapod/core/streams/view.py b/src/orcapod/core/streams/view.py index ed2cab6af..8737372c9 100644 --- a/src/orcapod/core/streams/view.py +++ b/src/orcapod/core/streams/view.py @@ -77,6 +77,12 @@ class ViewStatus(str, Enum): "not run; run() delivers its side effects; displaying never delivers side effects" ) +# An operator node whose output came from a load-only run: its input may lack rows that a +# full run computes (ITL-1095). +LOADED_ONLY_OPERATOR_NOTE = ( + "loaded only: the output of a load-only run, whose inputs may lack rows" +) + _DEFAULT_NOTES = { ViewStatus.NOT_COMPUTED: NOT_COMPUTED_NOTE, ViewStatus.NOTHING_STORED: NOTHING_STORED_NOTE, diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 3f720a19d..805965003 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -942,7 +942,7 @@ def run( plan = plan_run(prepared.exec_dag, load_only=load_only) counter = RunCountingObserver() effective_orchestrator = orchestrator or SyncPipelineOrchestrator() - with self._executing(prepared, plan, commit_every_rows): + with self._executing(prepared, plan, commit_every_rows, load_only=load_only): _ = effective_orchestrator.run( plan.dag, observer=_with_counter(observer, counter), @@ -1007,7 +1007,7 @@ async def run_async( # The state rule (ITL-1095): a node that holds a current run is not run again. plan = plan_run(prepared.exec_dag, load_only=load_only) counter = RunCountingObserver() - with self._executing(prepared, plan, commit_every_rows): + with self._executing(prepared, plan, commit_every_rows, load_only=load_only): _ = await orchestrator.run_async( plan.dag, observer=_with_counter(observer, counter), @@ -1122,6 +1122,21 @@ def _prepare_run( "original pipeline definition." ) + # A loaded job whose graph lost a node (PipelineJob.load does not restore + # side-effect nodes yet, ITL-1094) cannot feed the node below it. + if self._loaded: + for node in self._persistent_node_map.values(): + inputs = getattr(node, "upstreams", ()) + if isinstance(node, (FunctionJobNode, OperatorJobNode)) and any( + upstream is None for upstream in inputs + ): + raise RuntimeError( + f"Node {node.label!r} of the loaded job has lost an input node: " + "PipelineJob.load does not restore side-effect nodes yet " + "(ITL-1094), so the nodes below one cannot run, not even " + "load-only. Read their stored results with node.as_source()." + ) + # --- Build hash-keyed OrcaDAG for topological ordering + exclusion --- hash_dag: OrcaDAG[str] = OrcaDAG() for u, v in self._graph_edges: @@ -1180,7 +1195,9 @@ def _prepare_run( # this run writes. Those rows would otherwise count towards a table's # batch_size, which could then be reached mid-run and commit this run's pipeline # records ahead of their results (ITL-919). - if table_committing is not None: + # A load-only run writes nothing, so it commits nothing, not even pending writes + # of earlier operations (ITL-1095). + if table_committing is not None and not load_only: try: pipeline_db.flush() except StoreError as exc: @@ -1201,16 +1218,24 @@ def _prepare_run( @contextlib.contextmanager def _executing( - self, prepared: "_PreparedRun", plan: RunPlan, commit_every_rows: int + self, + prepared: "_PreparedRun", + plan: RunPlan, + commit_every_rows: int, + *, + load_only: bool = False, ) -> Iterator[None]: """Hold every function node to the run's commit cadence while the orchestrator runs. - A failure inside commits the writes that succeeded before it, then propagates. + A failure inside commits the writes that succeeded before it (not in a load-only + run, which writes nothing), clears every node the run ran (whether their writes + were committed is unknown), then propagates. Args: prepared: The run, from ``_prepare_run``. plan: The nodes the run runs (``plan_run``). commit_every_rows: As for ``run``. + load_only: As for ``run``. """ try: with contextlib.ExitStack() as batched: @@ -1224,7 +1249,9 @@ def _executing( except Exception as run_error: # Commit the writes that did succeed before the failure, so their rows # do not look dropped, then report the failure itself. - flush_best_effort(run_error, prepared.pipeline_db, prepared.result_db) + if not load_only: + flush_best_effort(run_error, prepared.pipeline_db, prepared.result_db) + clear_run_rows(plan.to_run) raise def _finish_run( @@ -1263,8 +1290,10 @@ def _finish_run( + "have been stored." ) try: - pipeline_db.flush() - result_db.flush() + if not load_only: + # A load-only run writes nothing and commits nothing (ITL-1095). + pipeline_db.flush() + result_db.flush() except StoreReadError as exc: # A stored table could not be read: report the read failure itself. exc.add_note(pending_note) diff --git a/src/orcapod/pipeline/node_run.py b/src/orcapod/pipeline/node_run.py index 946b24b1d..597618e1d 100644 --- a/src/orcapod/pipeline/node_run.py +++ b/src/orcapod/pipeline/node_run.py @@ -201,7 +201,9 @@ def run_with_dependencies(target: Any, *, load_only: bool) -> RunSummary: run and its rows feed the nodes after it, a stream outside the job is read, every other node runs once, in dependency order, on the ``SyncPipelineOrchestrator`` (error policy ``"continue"``). The run ends with a commit of every store it wrote - to, also, best-effort, when it fails. + to, also, best-effort, when it fails; a load-only run commits nothing (it writes + nothing, and pending writes of others stay pending). A run that raises leaves no + node it ran holding rows. Args: target: The job node whose ``run()`` this is. @@ -245,10 +247,16 @@ def visit(node: Any) -> None: plan.dag, observer=counter, load_only=load_only, held_rows=plan.held_rows ) except Exception as run_error: - # Commit the writes that succeeded before the failure, as job.run() does. - flush_best_effort(run_error, *stores) + if not load_only: + # Commit the writes that succeeded before the failure, as job.run() does; + # a load-only run writes nothing, so it commits nothing (ITL-1095). + flush_best_effort(run_error, *stores) + # A run that raised leaves no node it ran holding rows: whether their writes + # were committed is unknown. + clear_run_rows(plan.to_run) raise - commit_run(stores, plan.to_run) + if not load_only: + commit_run(stores, plan.to_run) return RunSummary( _job_name(target), run_node_summaries(graph, counter, held=plan.held_counts), diff --git a/src/orcapod/pipeline/run_summary.py b/src/orcapod/pipeline/run_summary.py index d003dde3d..975501c9d 100644 --- a/src/orcapod/pipeline/run_summary.py +++ b/src/orcapod/pipeline/run_summary.py @@ -152,6 +152,12 @@ def _headline(self) -> str: if self.target is None else f"node {self.target!r} and the nodes it depends on" ) + if self.nodes and all( + node.held and not (node.computed or node.reused or node.failed) + for node in self.nodes.values() + ): + run = "Load-only run" if self.load_only else "Run" + return f"{run} of {subject}: nothing to do; every node holds a current run." if self.load_only: skipped = sum(node.not_stored for node in self.nodes.values()) outcome = ( diff --git a/src/orcapod/side_effects.py b/src/orcapod/side_effects.py index 6da628ced..eda680270 100644 --- a/src/orcapod/side_effects.py +++ b/src/orcapod/side_effects.py @@ -1085,6 +1085,20 @@ def execute( rows = list(input_stream.iter_data()) self._hold(rows, load_only=True) return rows + try: + results = self._deliver_rows(input_stream, run_id=run_id) + except BaseException: + # A run that raised defines no rows: the node is stale, and so is every + # node below it (ITL-1095). + self.clear_cache() + raise + self._hold(results) + return results + + def _deliver_rows( + self, input_stream: StreamProtocol, *, run_id: str | None + ) -> list[tuple[TagProtocol, DataProtocol]]: + """Deliver every row of ``input_stream``; return the rows passed on.""" self._refuse_log_of_another_data_context() results: list[tuple[TagProtocol, DataProtocol]] = [] for tag, data in input_stream.iter_data(): @@ -1104,7 +1118,6 @@ def execute( ) if result is not None: results.append(result) - self._hold(results) return results def _hold( @@ -1226,7 +1239,8 @@ def run(self, *, load_only: bool = False) -> StreamView: def _run_alone(self, *, load_only: bool) -> None: """Run this node over an input stream that is not a job node; commit its log.""" - database = self._pipeline_database + # A load-only run writes nothing, so it commits nothing (ITL-1095). + database = None if load_only else self._pipeline_database try: _ = self.execute(self._input_stream, load_only=load_only) except Exception as run_error: @@ -1303,8 +1317,11 @@ async def async_execute( await output.send(item) # Held and stamped before the output closes (ITL-1042). self._hold(rows, load_only=True) + except BaseException: + self.clear_cache() + raise finally: - await output.close() + await self._close_output(output) return # This run's rows, in the order they were sent downstream. passed_on: list[tuple[TagProtocol, DataProtocol]] = [] @@ -1347,5 +1364,19 @@ async def process_one(tag: TagProtocol, data: DataProtocol) -> None: # Held and stamped before the output closes, so every downstream node, which # stamps after it has seen that close, is newer than this one (ITL-1042). self._hold(passed_on) + except BaseException: + # A run that raised or was cancelled defines no rows (ITL-1095). + self.clear_cache() + raise finally: + await self._close_output(output) + + async def _close_output( + self, output: WritableChannel[tuple[TagProtocol, DataProtocol]] + ) -> None: + """Close ``output``; a close that fails fails the run, which then holds no rows.""" + try: await output.close() + except BaseException: + self.clear_cache() + raise diff --git a/tests/test_core/function_pod/test_function_pod_node_stream.py b/tests/test_core/function_pod/test_function_pod_node_stream.py index bda5c9e06..8ba09ec66 100644 --- a/tests/test_core/function_pod/test_function_pod_node_stream.py +++ b/tests/test_core/function_pod/test_function_pod_node_stream.py @@ -395,7 +395,9 @@ def test_is_stale_true_after_upstream_modified(self, double_pf): input_stream=input_stream, pipeline_database=db, ) - list(node.iter_data()) + node.run() # a node that has not run is stale already (ITL-1095) + assert not node.is_stale + assert len(list(node.iter_data())) == 3 time.sleep(0.01) input_stream._update_modified_time() @@ -505,14 +507,20 @@ def test_as_table_auto_detects_stale_and_repopulates(self, double_pf): def test_no_auto_clear_when_not_stale(self, double_pf): node = _make_node(double_pf, n=3) + node.run() # a node that has not run holds nothing (ITL-1095) + assert not node.is_stale list(node.iter_data()) cached_count = len(node._cached_output_datas) + assert cached_count == 3 list(node.iter_data()) assert len(node._cached_output_datas) == cached_count def test_as_table_no_auto_clear_when_not_stale(self, double_pf): node = _make_node(double_pf, n=3) + node.run() # a node that has not run holds nothing (ITL-1095) + assert not node.is_stale table_before = node.as_table() + assert table_before.num_rows == 3 table_after = node.as_table() assert table_before.equals(table_after) diff --git a/tests/test_pipeline/test_current_run_review_fixes.py b/tests/test_pipeline/test_current_run_review_fixes.py new file mode 100644 index 000000000..14c3619d2 --- /dev/null +++ b/tests/test_pipeline/test_current_run_review_fixes.py @@ -0,0 +1,490 @@ +"""Review fixes for the current-run rule and load-only runs (ITL-1095, PR #380 review). + +Each test names the guarantee it protects: a run that raised (in its observer, its store, +its commit or its output) leaves no node holding its rows; a load-only run writes nothing, +not even pending writes of others; an operator filled by a load-only run says so; the +freshness check stays linear on a DAG of shared ancestors. +""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from typing import Any +from unittest import mock + +import pyarrow as pa +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.core.nodes import OperatorJobNode +from orcapod.core.nodes.source_node import SourceJobNode +from orcapod.core.operators import Join, MapData, SelectDataColumns +from orcapod.core.streams import ArrowTableStream, ViewStatus +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.errors import StoreWriteError +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.pipeline.observer import NoOpObserver +from orcapod.side_effects import SideEffectPod, SideEffectPodConfig, side_effect_pod +from orcapod.sources import ArrowTableSource, DictSource +from orcapod.types import CacheMode + +CALLS: list[str] = [] +FAIL = {"tap": False} + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + """Return ``x + 1``.""" + CALLS.append("inc") + return x + 1 + + +@function_pod(output_keys="z") +def dbl(y: int) -> int: + """Return ``2 * y``.""" + CALLS.append("dbl") + return 2 * y + + +@function_pod(output_keys="w") +def neg(z: int) -> int: + """Return ``-z``.""" + CALLS.append("neg") + return -z + + +@function_pod(output_keys="z") +def fails_on_21(y: int) -> int: + """Return ``2 * y``; raise for ``y == 21``.""" + if y == 21: + raise ValueError("y is 21") + return 2 * y + + +@function_pod(output_keys="total") +def total(y: list[int]) -> int: + """Return the sum of a group's values.""" + return sum(y) + + +def _deliver(y: int, ctx: Any) -> None: + """A side effect that fails while ``FAIL['tap']`` is set.""" + if FAIL["tap"]: + raise RuntimeError("delivery failed") + + +ROWS = [{"g": i % 2, "k": i, "x": 10 * i} for i in range(6)] +ORCHESTRATORS = {"sync": SyncPipelineOrchestrator, "async": AsyncPipelineOrchestrator} + + +@pytest.fixture(autouse=True) +def _reset() -> None: + CALLS.clear() + FAIL["tap"] = False + + +def _source(data: list[dict[str, int]] = ROWS) -> DictSource: + return DictSource(data, tag_columns=["g", "k"], source_id="s") + + +def _grouped_job(data: list[dict[str, int]], db: Any) -> PipelineJob: + """``rowwise`` -> ``grouped`` (GroupBy) -> ``summed``.""" + job = PipelineJob(name="j", database=db) + with job: + rows: Any = inc.pod(_source(data), label="rowwise") + total.pod(rows.group_by("g", label="grouped"), label="summed") + return job + + +def _chain_job(data: list[dict[str, int]], db: Any) -> PipelineJob: + """``a`` (inc) -> ``b`` (dbl) -> ``c`` (neg).""" + job = PipelineJob(name="c", database=db) + with job: + neg.pod(dbl.pod(inc.pod(_source(data), label="a"), label="b"), label="c") + return job + + +class FailsAtNodeEnd(NoOpObserver): + """An observer whose ``on_node_end`` raises for one node.""" + + def __init__(self, label: str) -> None: + self.label = label + + def on_node_end(self, node_label: str, node_hash: str) -> None: + """Raise at the node's end, as a broken observer would.""" + if node_label == self.label: + raise RuntimeError("observer failed") + + def contextualize(self, *identity_path: str) -> FailsAtNodeEnd: + """Keep this observer for every node.""" + return self + + +# --------------------------------------------------------------------------- +# A run that raised leaves no node holding its rows +# --------------------------------------------------------------------------- + + +def test_an_observer_failing_at_the_end_of_a_load_only_node_leaves_it_empty() -> None: + db = InMemoryArrowDatabase() + _grouped_job(ROWS, db).run() + job = _grouped_job(ROWS, db) + with pytest.raises(RuntimeError, match="observer failed"): + job.run(load_only=True, observer=FailsAtNodeEnd("rowwise")) + assert job.nodes["rowwise"].as_table().num_rows == 0 + assert job.nodes["rowwise"].is_stale + + +def test_a_failed_commit_of_a_node_run_alone_leaves_it_empty() -> None: + from orcapod.core.nodes import FunctionJobNode + + table = pa.table( + {"k": [0, 1], "x": [1, 2]}, + schema=pa.schema( + [ + pa.field("k", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] + ), + ) + node = FunctionJobNode( + inc.pod, # pyright: ignore[reportArgumentType] # the decorator's pod type is not declared as FunctionPodProtocol + ArrowTableStream(table, tag_columns=["k"]), + pipeline_database=InMemoryArrowDatabase(), + ) + + def fail(self: Any) -> None: + raise OSError("disk full") + + with ( + mock.patch.object(InMemoryArrowDatabase, "flush", fail), + pytest.raises(StoreWriteError), + ): + node.run() + assert node.as_table().num_rows == 0 + assert node.is_stale + + +def test_a_failed_run_clears_every_node_it_ran() -> None: + """A downstream failure (fail_fast) leaves the upstream node that ran empty too. + + Sync only: the asynchronous orchestrator does not stop for ``fail_fast`` (ITL-879). + """ + job = PipelineJob(name="f", database=InMemoryArrowDatabase()) + with job: + fails_on_21.pod(inc.pod(_source(), label="up"), label="down") + with pytest.raises(ValueError, match="y is 21"): + job.run(orchestrator=SyncPipelineOrchestrator(error_policy="fail_fast")) + assert job.nodes["up"].as_table().num_rows == 0 + assert job.nodes["up"].is_stale + + +def test_a_failed_cascade_clears_every_node_it_ran() -> None: + job = PipelineJob(name="f", database=InMemoryArrowDatabase()) + with job: + fails_on_21.pod(inc.pod(_source(), label="up"), label="down") + + def fail(self: Any) -> None: + raise OSError("disk full") + + with ( + mock.patch.object(InMemoryArrowDatabase, "flush", fail), + pytest.raises(StoreWriteError), + ): + job.nodes["down"].run() + for label in ("up", "down"): + assert job.nodes[label].as_table().num_rows == 0, label + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_a_failed_operator_store_leaves_the_operator_empty(orchestrator: str) -> None: + db = InMemoryArrowDatabase() + job = PipelineJob(name="o", database=db) + with job: + SelectDataColumns(["x"])(_source(), label="sel") + job.nodes["sel"].attach_databases( + pipeline_database=db.at("o"), cache_mode=CacheMode.LOG + ) + with ( + mock.patch.object( + InMemoryArrowDatabase, "add_records", side_effect=OSError("disk full") + ), + pytest.raises((StoreWriteError, ExceptionGroup)), + ): + job.run(orchestrator=ORCHESTRATORS[orchestrator]()) + assert job.nodes["sel"].as_table().num_rows == 0 + assert job.nodes["sel"].is_stale + + +def _tap_job(db: Any) -> PipelineJob: + """``rows`` -> ``tap`` (side effect, raises while FAIL is set) -> ``down``.""" + tap = side_effect_pod(_deliver, config=SideEffectPodConfig(track_completion=False)) + assert isinstance(tap, SideEffectPod) + job = PipelineJob(name="t", database=db) + with job: + tapped: Any = tap.process(inc.pod(_source(), label="rows"), label="tap") + dbl.pod(tapped.select_data_columns("y"), label="down") + return job + + +def test_a_failed_side_effect_run_makes_the_nodes_below_it_stale() -> None: + job = _tap_job(InMemoryArrowDatabase()) + job.run() + assert job.nodes["down"].as_table().num_rows == 6 + job.nodes["tap"].clear_cache() + FAIL["tap"] = True + with pytest.raises(RuntimeError, match="delivery failed"): + job.nodes["tap"].run() + assert job.nodes["tap"].is_stale + assert job.nodes["down"].is_stale + assert job.nodes["down"].as_table().num_rows == 0 + + +def test_a_side_effect_node_whose_output_cannot_close_holds_nothing() -> None: + from orcapod.channels import Channel + + job = _tap_job(InMemoryArrowDatabase()) + job.run() + node = job.nodes["tap"] + node.clear_cache() + + class CloseFails: + """An output writer that keeps what it is sent and raises on close.""" + + async def send(self, item: Any) -> None: + """Accept ``item``.""" + + async def close(self) -> None: + """Raise, as a broken channel would.""" + raise OSError("output cannot be closed") + + async def run() -> None: + inputs: Channel[Any] = Channel(buffer_size=16) + for item in job.nodes["rows"].iter_data(): + await inputs.writer.send(item) + await inputs.writer.close() + # The node's channel entry point is what this test is about. + await node.async_execute([inputs.reader], CloseFails()) # pyright: ignore[reportArgumentType] # a test double of the writer + + with pytest.raises(OSError, match="cannot be closed"): + asyncio.run(run()) + assert node.as_table().num_rows == 0 + assert node.is_stale + + +# --------------------------------------------------------------------------- +# A load-only run writes nothing, pending writes of others included +# --------------------------------------------------------------------------- + + +def _pending(db: InMemoryArrowDatabase) -> dict[str, int]: + """The store's pending batches with their row counts (private on purpose: what a + load-only run must leave as it found it).""" + return {key: table.num_rows for key, table in db._pending_batches.items()} # pyright: ignore[reportPrivateUsage] # the pending batches are what this test is about + + +def _add_pending_write(db: InMemoryArrowDatabase) -> None: + db.add_records( + ("elsewhere",), + pa.table({"__rid": [b"r1"], "v": [1]}), + record_id_column="__rid", + ) + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_a_load_only_run_leaves_pending_writes_as_it_found_them( + orchestrator: str, +) -> None: + db = InMemoryArrowDatabase() + _grouped_job(ROWS, db).run() + _add_pending_write(db) + before = _pending(db) + assert before + job = _grouped_job(ROWS, db) + with mock.patch.object( + InMemoryArrowDatabase, "flush", side_effect=AssertionError("flushed") + ): + job.run(orchestrator=ORCHESTRATORS[orchestrator](), load_only=True) + cascade = _grouped_job(ROWS, db) + cascade.nodes["summed"].run(load_only=True) + assert _pending(db) == before + assert job.nodes["summed"].as_table().num_rows == 2 + + +# --------------------------------------------------------------------------- +# Operators: the standalone LOG write is committed; load-only output is marked +# --------------------------------------------------------------------------- + + +def _source_table() -> ArrowTableSource: + table = pa.table( + {"k": [0, 1, 2], "x": [1, 2, 3]}, + schema=pa.schema( + [ + pa.field("k", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] + ), + ) + return ArrowTableSource(table, tag_columns=["k"], source_id="t") + + +def test_a_standalone_log_run_commits_its_write(tmp_path: Path) -> None: + node = OperatorJobNode( + operator=SelectDataColumns(["x"]), + input_streams=[_source_table()], + pipeline_database=DeltaTableDatabase(tmp_path), + cache_mode=CacheMode.LOG, + ) + node.run() + reopened = OperatorJobNode( + operator=SelectDataColumns(["x"]), + input_streams=[_source_table()], + pipeline_database=DeltaTableDatabase(tmp_path), + cache_mode=CacheMode.LOG, + ) + stored = reopened.get_all_records() + assert stored is not None and stored.num_rows == 3 + + +def test_a_standalone_log_run_after_a_load_only_run_stores(tmp_path: Path) -> None: + db = InMemoryArrowDatabase() + node = OperatorJobNode( + operator=SelectDataColumns(["x"]), + input_streams=[_source_table()], + pipeline_database=db, + cache_mode=CacheMode.LOG, + ) + node.run(load_only=True) + assert node.get_all_records() is None + node.run() + stored = node.get_all_records() + assert stored is not None and stored.num_rows == 3 + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_an_operator_filled_by_a_load_only_run_says_so(orchestrator: str) -> None: + db = InMemoryArrowDatabase() + _grouped_job(ROWS, db).run() + job = _grouped_job(ROWS, db) + job.run(orchestrator=ORCHESTRATORS[orchestrator](), load_only=True) + grouped = job.nodes["grouped"] + assert grouped.head().status is ViewStatus.LOADED_ONLY + assert grouped.run(load_only=True).status is ViewStatus.LOADED_ONLY + + +def test_an_unrun_loaded_operator_without_its_operator_keeps_its_columns( + tmp_path: Path, +) -> None: + db = InMemoryArrowDatabase() + job = PipelineJob(name="l", database=db) + with job: + Join()( + DictSource([{"k": 0, "a": 1}], tag_columns=["k"], source_id="a"), + DictSource([{"k": 0, "b": 2}], tag_columns=["k"], source_id="b"), + label="joined", + ) + job.run() + job.save(tmp_path / "job.json") + nodes = json.loads((tmp_path / "job.json").read_text())["pipeline"]["nodes"] + (descriptor,) = [d for d in nodes.values() if d.get("label") == "joined"] + # A node rebuilt without its operator is what this test is about. + stub = OperatorJobNode.from_descriptor( + descriptor, operator=None, input_streams=(), databases={} + ) + table = stub.as_table() + assert table.num_rows == 0 + assert sorted(table.column_names) == ["a", "b", "k"] + + +# --------------------------------------------------------------------------- +# Staleness and summaries +# --------------------------------------------------------------------------- + + +def test_a_bound_source_with_no_modified_time_makes_its_slot_unknown() -> None: + stream = ArrowTableStream(pa.table({"k": [0], "x": [1]}), tag_columns=["k"]) + slot = SourceJobNode(name="s", bound_source=stream) + assert slot.last_modified is not None + stream._set_modified_time(None) # pyright: ignore[reportPrivateUsage] # an unknown modified time is what this test is about + assert slot.last_modified is None + + +def test_freshness_of_shared_ancestors_is_checked_once_per_traversal() -> None: + """Five split-and-join stages: the check visits each node once, not once per path. + + (Five stages: building a deeper job of this shape is itself slow, independently of + this check.) + """ + from orcapod.core.streams.base import StreamBase + + job = PipelineJob(name="d", database=InMemoryArrowDatabase()) + with job: + current: Any = inc.pod(_source(), label="start") + for stage in range(5): + left = MapData({"y": f"l{stage}"})(current, label=f"left{stage}") + right = MapData({"y": f"r{stage}"})(current, label=f"right{stage}") + joined = Join()(left, right, label=f"join{stage}") + picked = SelectDataColumns([f"l{stage}"])(joined, label=f"pick{stage}") + current = MapData({f"l{stage}": "y"})(picked, label=f"back{stage}") + job.run() + original = StreamBase.is_stale.fget + assert original is not None + calls = 0 + + def counting(self: Any) -> bool: + nonlocal calls + calls += 1 + return original(self) + + with mock.patch.object(StreamBase, "is_stale", property(counting)): + assert job.nodes["back4"].holds_current_run() + # One own-staleness check per job node of the ancestry (26 nodes), not per path. + assert calls <= 26 + + +def test_a_held_dependency_keeps_its_incomplete_inputs_in_the_summary() -> None: + """A -> B -> C, load-only: A skipped inputs; C cleared and loaded again still names A.""" + db = InMemoryArrowDatabase() + _chain_job(ROWS[:3], db).run() + job = _chain_job(ROWS, db) + job.run(load_only=True) + job.nodes["c"].clear_cache() + summary = job.run(load_only=True) + assert summary.nodes["c"].inputs_incomplete == ("a",) + + +def test_a_run_with_every_node_held_says_so() -> None: + job = _grouped_job(ROWS, InMemoryArrowDatabase()) + job.run() + summary = job.run() + assert repr(summary).splitlines()[0] == ( + "Run of job 'j': nothing to do; every node holds a current run." + ) + + +# --------------------------------------------------------------------------- +# A loaded job missing a node (ITL-1094) fails load-only with a clear error +# --------------------------------------------------------------------------- + + +def _deliver_x(x: int, ctx: Any) -> None: + """A side effect over the source's own column.""" + + +def test_a_loaded_job_without_its_side_effect_node_names_it(tmp_path: Path) -> None: + tap = side_effect_pod(_deliver_x) + assert isinstance(tap, SideEffectPod) + db = InMemoryArrowDatabase() + job = PipelineJob(name="m", database=db) + with job: + tapped: Any = tap.process(_source(), label="delivered") + inc.pod(tapped, label="down") + job.run() + job.save(tmp_path / "job.json") + loaded = PipelineJob.load(tmp_path / "job.json", database=db) + loaded.bind(sources={"s": _source()}) + with pytest.raises(RuntimeError, match=r"'down'.*ITL-1094"): + loaded.run(load_only=True) From 075b8e90fa27c903b146f4ab9903c2b898218b22 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 03:10:32 -0800 Subject: [PATCH 32/63] fix(pipeline): second review round for the current-run rule (ITL-1095) CodeRabbit review of b1b99445 and 3cd6bef5, plan approved by the coordinator: - a run that left failed rows (error_policy continue) is never current: the next run computes only the failed rows and reads the rest (sync and async); - a run that runs no node touches no store: no pre-run or end-of-run flush; - a held load-only node keeps its missing inputs in the summary; - an interrupted or cancelled run clears the nodes it ran (job.run and cascade); - a loaded node whose saved inputs were not all restored raises naming ITL-1094 (checked against the saved graph, in one place); - an async CACHE_ONLY node holds the rows it sent; - docs/changelog: only the nodes that ran are cleared; clear each branch or rebind; failed rows are retried; docstrings for the planner helpers and tests. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1095.md | 2 +- docs/concepts/function-pods.md | 8 +- src/orcapod/core/nodes/function_node.py | 55 +++++- src/orcapod/pipeline/job.py | 55 +++--- src/orcapod/pipeline/node_run.py | 8 +- src/orcapod/pipeline/run_summary.py | 16 +- .../test_current_run_review_fixes.py | 159 ++++++++++++++++++ .../test_pipeline/test_job_run_state_rule.py | 12 ++ 8 files changed, 279 insertions(+), 36 deletions(-) diff --git a/changelog.d/ITL-1095.md b/changelog.d/ITL-1095.md index 4629ac44b..c6ab32168 100644 --- a/changelog.d/ITL-1095.md +++ b/changelog.d/ITL-1095.md @@ -6,7 +6,7 @@ - A job node's `run()` is state-driven and runs what the node depends on (developer, November 13, 2026; ITL-1095). A node that holds a completed run and is not stale runs nothing: `run()` returns the rows it holds. Otherwise every job node it depends on that holds no current run runs first, each once, as `job.run()` would run that part of the job, then the node; the run's writes are committed when it ends (before, `node.run()` read an upstream node's stored rows, or nothing, and left its last pipeline record and a `LOG` operator's output uncommitted). `run()` of a node whose function pod is a placeholder, or that was loaded without one, raises unless `load_only=True`; a `CACHE_ONLY` node's `run()` raises (its input is unavailable): read its stored results with `as_source()`. A job node's `run()` takes one keyword-only argument, `load_only`; every other stream's `run()` still takes none. - A side-effect job node follows the same rule: it is stale before a run, and while it holds a current run its `run()` (or a downstream node's `run()`) delivers nothing; to deliver again without `track_completion`, call `clear_cache()` on it first (`job.run()` still delivers as before). A load-only run passes its rows on and delivers nothing. - `job.run()` and `job.run_async()` follow the same rule as `node.run()` (developer, November 14, 2026): a node that holds a current run is not run again, and its rows feed the nodes after it. Running an unchanged job a second time now computes nothing, reads and writes nothing in the store, does not read its sources and delivers no side effect, even without `track_completion` (before, every `job.run()` re-read every source and re-processed every node, and delivered every untracked side effect again). Nodes run when they have not run in this process, were cleared, are below a cleared node or a node that ran again, or below a source bound again or whose own modified time moved. After this change a re-run does not notice new data in a source whose object is unchanged (a file or table changed in place, a polling source with new rows): bind a new source object, or call `clear_cache()` on the first node below it, to read it again (a start-of-run check for new data is ITL-1102). The summary reports a node that was not run as held: "rowwise: 6 rows: held from the previous run" (`NodeRunSummary.held`). If the commit at the end of a run fails, the nodes it ran are cleared and the nodes it held keep their rows. -- If the commit at the end of `job.run()` or `node.run()` fails, every function and operator node of the run is cleared (not run, stale) before the error propagates. +- If a run raises or is interrupted, or the commit at the end of `job.run()` or `node.run()` fails, every node the run ran is cleared (not run, stale) before the error propagates; nodes it held keep their rows. A run that left failed rows (`error_policy="continue"`) is never current: the next run computes only the failed rows and reads the rest from the store. ### Added diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index aa183b589..3479f9251 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -598,6 +598,10 @@ nothing, also on the asynchronous orchestrator, and the summary says so per node run's rows are never held by a full run, nor are the rows of any node below them. A source is read only when a node that runs reads it. +A node whose last run left failed rows (with the default `error_policy="continue"`) is not +held either: the next run computes only the failed rows and reads the rest from the store, +so running the job again after fixing a transient failure retries exactly those rows. + A node is stale, and runs, when: - it has not run in this process, its last run raised, or the commit at the end of its run @@ -611,8 +615,8 @@ Staleness comes from modified times only. A source whose rows change while its o the same (a CSV or Delta file rewritten in place, a database table that grew, a list a `DictSource` was built from and that was changed afterwards, a polling source with new rows) is not seen: a run holds the nodes below it. To read it again, build a new source -object and bind it (`job.bind(sources=...)`), or call `clear_cache()` on the first node -below it. (A start-of-run check for new data is planned, ITL-1102.) To deliver a +object and bind it (`job.bind(sources=...)`), which makes every node that reads it stale, +or call `clear_cache()` on each node that reads it directly (one per branch). (A start-of-run check for new data is planned, ITL-1102.) To deliver a side effect again (with `track_completion=False`), call `clear_cache()` on its node first. `node.last_run` holds the counts of a `node.run()`, as `job.last_run` does for `job.run()`, diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 74cad51f2..55134428b 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -1106,6 +1106,9 @@ class FunctionJobNode(RunHoldingNode, FunctionNodeBase): # stored result (ITL-1095). Class defaults for nodes rebuilt from a descriptor. _run_load_only: bool = False _run_not_stored: int = 0 + # Rows of the last completed run whose function raised (error_policy "continue"): + # such a run is never current, so the next run retries them (ITL-1095). + _run_failed: int = 0 def __init__( self, @@ -1572,6 +1575,7 @@ def clear_cache(self) -> None: self._run_entry_counts = None self._run_load_only = False self._run_not_stored = 0 + self._run_failed = 0 # No content established: the node, and every node downstream, is stale. self._set_modified_time(None) @@ -1599,7 +1603,12 @@ def _held_pairs(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: yield tag, data def _set_run_rows( - self, entry_ids: list[bytes], *, load_only: bool = False, not_stored: int = 0 + self, + entry_ids: list[bytes], + *, + load_only: bool = False, + not_stored: int = 0, + failed: int = 0, ) -> None: """Make the inputs of a completed run, by base entry id, the rows this node holds. @@ -1607,17 +1616,26 @@ def _set_run_rows( entry_ids: The base entry ids of the run's inputs whose rows the node holds. load_only: Whether the run was load-only (ITL-1095). not_stored: How many inputs of a load-only run had no stored result. + failed: How many rows of the run failed (error policy ``"continue"``). """ self._run_entry_counts = Counter(entry_ids) self._run_load_only = load_only self._run_not_stored = not_stored + self._run_failed = failed self._cached_output_table = None self._cached_content_hash_column = None def _holds_run(self, *, load_only: bool) -> bool: - """Whether the node holds a completed run; a load-only one only for ``load_only``.""" - return self._run_entry_counts is not None and ( - load_only or not self._run_load_only + """Whether the node holds a completed run that a run need not redo. + + A load-only run's rows are good only for another load-only run. A run that left + failed rows is never good: the next run retries them, reading the rest from the + store (ITL-1095). + """ + return ( + self._run_entry_counts is not None + and self._run_failed == 0 + and (load_only or not self._run_load_only) ) @property @@ -2204,6 +2222,7 @@ def execute( # Base entry ids of the inputs that gave a row (a cache hit or a # successful computation), one per input: the node's rows afterwards. emitted_entry_ids: list[bytes] = [] + failed_rows = 0 for tag, data, base_entry_id in upstream_entries: ctx_obs.on_data_start(node_label, tag, data) @@ -2257,6 +2276,7 @@ def execute( if error_policy == "fail_fast": ctx_obs.on_node_end(node_label, node_hash) raise + failed_rows += 1 continue ctx_obs.on_data_end(node_label, tag, data, result, cached=False) @@ -2271,7 +2291,7 @@ def execute( self._commit_at_node_end, ctx_obs, node_label, node_hash ) # The node's rows are those of this run's inputs, whatever else it caches. - self._set_run_rows(emitted_entry_ids) + self._set_run_rows(emitted_entry_ids, failed=failed_rows) ctx_obs.on_node_end(node_label, node_hash) except Exception as run_error: self._commit_after_failure(run_error) @@ -3350,14 +3370,27 @@ async def _async_execute_cache_only( self._cached_output_table = None self._cached_content_hash_column = None - for tag, data in self._cached_output_datas.values(): - if data is not None: + sent: list[bytes] = [] + for entry_id, (tag, data) in list(self._cached_output_datas.items()): + if data is not None and not isinstance(data, EmptyData): ctx_obs.on_data_start(node_label, tag, data) ctx_obs.on_data_end(node_label, tag, data, data, cached=True) await output.send((tag, data)) + sent.append(entry_id) + # The node holds the rows it sent: a completed load of what is stored + # (ITL-1095), stamped before the output closes (ITL-1042). + self._set_run_rows(sent, load_only=True) + self._update_modified_time() ctx_obs.on_node_end(node_label, node_hash) + except BaseException: + self.clear_cache() + raise finally: - await output.close() + try: + await output.close() + except BaseException: + self.clear_cache() + raise # ------------------------------------------------------------------ # Head @@ -3732,6 +3765,8 @@ async def async_execute( # Base entry ids of this run's inputs that gave a row (a cache hit or a # forwarded computed row), one per input (ITL-1093). run_entry_ids: list[bytes] = [] + # Rows whose function raised (error policy "continue"). + failed_rows: list[int] = [0] async def route_inputs() -> None: """Stage 1: send cache hits to output; stamp misses for computation.""" @@ -3890,6 +3925,8 @@ def on_data_crash( # The pod re-raises a failed store read or write after # this call; name the node it belongs to. error.add_context(node_label=node_label) + else: + failed_rows[0] += 1 ctx_obs.on_data_crash(node_label, clean, data, error) # Clean up the dangling input_store entry so it doesn't # accumulate memory during long-running calls with errors. @@ -3920,7 +3957,7 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: self._commit_at_node_end() # The node's rows are those of this run's inputs, as the sync path's: # the cache also holds every stored result loaded above. - self._set_run_rows(run_entry_ids) + self._set_run_rows(run_entry_ids, failed=failed_rows[0]) else: # No-DB path: delegate dispatch entirely to the pod. The pod closes # the writer it is given; that close waits for the finally below, so diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 805965003..bd254eff2 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -69,6 +69,7 @@ class _PreparedRun: pipeline_uri: ``@``, passed to the observer. pipeline_db: The job's view of the store. result_db: The job's function-result view of the store. + plan: What the run runs and holds (``plan_run``, ITL-1095). """ exec_dag: OrcaDAG[Any] @@ -76,6 +77,7 @@ class _PreparedRun: pipeline_uri: str pipeline_db: "ArrowDatabaseProtocol" result_db: "ArrowDatabaseProtocol" + plan: "RunPlan" def _with_counter( @@ -939,7 +941,7 @@ def run( prepared = self._prepare_run(commit_every_rows, load_only=load_only) # The state rule (ITL-1095): a node that holds a current run is not run again. - plan = plan_run(prepared.exec_dag, load_only=load_only) + plan = prepared.plan counter = RunCountingObserver() effective_orchestrator = orchestrator or SyncPipelineOrchestrator() with self._executing(prepared, plan, commit_every_rows, load_only=load_only): @@ -1005,7 +1007,7 @@ async def run_async( ) prepared = self._prepare_run(commit_every_rows, load_only=load_only) # The state rule (ITL-1095): a node that holds a current run is not run again. - plan = plan_run(prepared.exec_dag, load_only=load_only) + plan = prepared.plan counter = RunCountingObserver() with self._executing(prepared, plan, commit_every_rows, load_only=load_only): _ = await orchestrator.run_async( @@ -1123,18 +1125,26 @@ def _prepare_run( ) # A loaded job whose graph lost a node (PipelineJob.load does not restore - # side-effect nodes yet, ITL-1094) cannot feed the node below it. + # side-effect nodes yet, ITL-1094) cannot feed the nodes below it: every saved + # input of a node must be a restored node. if self._loaded: - for node in self._persistent_node_map.values(): - inputs = getattr(node, "upstreams", ()) - if isinstance(node, (FunctionJobNode, OperatorJobNode)) and any( - upstream is None for upstream in inputs - ): + for node_hash, node in self._persistent_node_map.items(): + if not isinstance(node, (FunctionJobNode, OperatorJobNode)): + continue + if node_hash not in self._hash_graph: + continue + missing = [ + upstream + for upstream in self._hash_graph.predecessors(node_hash) + if upstream not in self._persistent_node_map + ] + if missing: raise RuntimeError( - f"Node {node.label!r} of the loaded job has lost an input node: " - "PipelineJob.load does not restore side-effect nodes yet " - "(ITL-1094), so the nodes below one cannot run, not even " - "load-only. Read their stored results with node.as_source()." + f"Node {node.label!r} of the loaded job has lost " + f"{len(missing)} of its input nodes: PipelineJob.load does not " + "restore side-effect nodes yet (ITL-1094), so the nodes below " + "one cannot run, not even load-only. Read their stored results " + "with node.as_source()." ) # --- Build hash-keyed OrcaDAG for topological ordering + exclusion --- @@ -1195,9 +1205,11 @@ def _prepare_run( # this run writes. Those rows would otherwise count towards a table's # batch_size, which could then be reached mid-run and commit this run's pipeline # records ahead of their results (ITL-919). - # A load-only run writes nothing, so it commits nothing, not even pending writes - # of earlier operations (ITL-1095). - if table_committing is not None and not load_only: + # The state rule (ITL-1095): a node that holds a current run is not run again. + plan = plan_run(exec_dag, load_only=load_only) + # A load-only run writes nothing, and a run that runs no node touches no store, + # so neither commits anything, not even pending writes of earlier operations. + if table_committing is not None and not load_only and plan.to_run: try: pipeline_db.flush() except StoreError as exc: @@ -1214,6 +1226,7 @@ def _prepare_run( pipeline_uri=pipeline_uri, pipeline_db=pipeline_db, result_db=result_db, + plan=plan, ) @contextlib.contextmanager @@ -1246,10 +1259,11 @@ def _executing( node._batched_commits(commit_every_rows) # pyright: ignore[reportPrivateUsage] # the job owns its nodes' commit cadence ) yield - except Exception as run_error: + except BaseException as run_error: # Commit the writes that did succeed before the failure, so their rows - # do not look dropped, then report the failure itself. - if not load_only: + # do not look dropped, then report the failure itself. A cancelled or + # interrupted run commits nothing more, but clears its nodes too. + if isinstance(run_error, Exception) and not load_only: flush_best_effort(run_error, prepared.pipeline_db, prepared.result_db) clear_run_rows(plan.to_run) raise @@ -1290,8 +1304,9 @@ def _finish_run( + "have been stored." ) try: - if not load_only: - # A load-only run writes nothing and commits nothing (ITL-1095). + if not load_only and plan.to_run: + # A load-only run writes nothing and commits nothing, and a run that + # ran no node touches no store (ITL-1095). pipeline_db.flush() result_db.flush() except StoreReadError as exc: diff --git a/src/orcapod/pipeline/node_run.py b/src/orcapod/pipeline/node_run.py index 597618e1d..c79cda5ec 100644 --- a/src/orcapod/pipeline/node_run.py +++ b/src/orcapod/pipeline/node_run.py @@ -132,14 +132,18 @@ def plan_run(graph: Any, *, load_only: bool, outside: Sequence[Any] = ()) -> Run order = list(graph.topological_sort()) def is_outside(node: Any) -> bool: + """Whether ``node`` is one of the ``outside`` streams: by identity, not by + equality (two equal streams are still two inputs).""" return any(node is other for other in outside) def is_current(node: Any) -> bool: + """Whether ``node`` holds a run this run need not redo (it is held).""" return isinstance(node, RunHoldingNode) and node.holds_current_run( load_only=load_only ) def is_source(node: Any) -> bool: + """Whether ``node`` is a source slot: read when a node that runs reads it.""" return isinstance(node, SourceJobNode) current = [node for node in order if is_current(node)] @@ -246,8 +250,8 @@ def visit(node: Any) -> None: _ = SyncPipelineOrchestrator().run( plan.dag, observer=counter, load_only=load_only, held_rows=plan.held_rows ) - except Exception as run_error: - if not load_only: + except BaseException as run_error: + if isinstance(run_error, Exception) and not load_only: # Commit the writes that succeeded before the failure, as job.run() does; # a load-only run writes nothing, so it commits nothing (ITL-1095). flush_best_effort(run_error, *stores) diff --git a/src/orcapod/pipeline/run_summary.py b/src/orcapod/pipeline/run_summary.py index 975501c9d..e3bc3dac2 100644 --- a/src/orcapod/pipeline/run_summary.py +++ b/src/orcapod/pipeline/run_summary.py @@ -249,7 +249,10 @@ def _load_only_html(self) -> str: def _counts(node: NodeRunSummary, load_only: bool) -> str: """A node's counts as text: what a full run did, or what a load-only run loaded.""" if node.held and not (node.computed or node.reused or node.failed): - return "held from the previous run" + text = "held from the previous run" + if node.not_stored: + text += f" ({node.not_stored} inputs without a stored result)" + return text if not load_only: return ( f"{node.computed} computed, {node.reused} from store, {node.failed} failed" @@ -454,9 +457,18 @@ def run_node_summaries( ) label: str = getattr(node, "label") # every graph node is labelled if held_rows is not None: + # A held node keeps what its run skipped: a load-only run's missing + # inputs are still missing. summaries.append( NodeRunSummary( - label, 0, 0, 0, None, inputs_incomplete=labels, held=held_rows + label, + 0, + 0, + 0, + None, + not_stored=getattr(node, "last_run_not_stored", 0), + inputs_incomplete=labels, + held=held_rows, ) ) continue diff --git a/tests/test_pipeline/test_current_run_review_fixes.py b/tests/test_pipeline/test_current_run_review_fixes.py index 14c3619d2..8c950bb57 100644 --- a/tests/test_pipeline/test_current_run_review_fixes.py +++ b/tests/test_pipeline/test_current_run_review_fixes.py @@ -488,3 +488,162 @@ def test_a_loaded_job_without_its_side_effect_node_names_it(tmp_path: Path) -> N loaded.bind(sources={"s": _source()}) with pytest.raises(RuntimeError, match=r"'down'.*ITL-1094"): loaded.run(load_only=True) + + +# --------------------------------------------------------------------------- +# Second review round +# --------------------------------------------------------------------------- + +FLAKY = {"fail": True} + + +@function_pod(output_keys="y") +def flaky(x: int) -> int: + """Return ``x + 1``; raise for ``x == 20`` while ``FLAKY['fail']`` is set.""" + CALLS.append("flaky") + if x == 20 and FLAKY["fail"]: + raise ValueError("transient failure") + return x + 1 + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_a_run_with_failed_rows_is_retried_by_the_next_run(orchestrator: str) -> None: + """A run that left failed rows is never current: the next run computes exactly + the failed rows and reads the rest; a third, unchanged run runs nothing.""" + FLAKY["fail"] = True + job = PipelineJob(name="r", database=InMemoryArrowDatabase()) + with job: + flaky.pod(_source(), label="node") + with pytest.warns(Warning): + job.run(orchestrator=ORCHESTRATORS[orchestrator]()) + assert job.nodes["node"].as_table().num_rows == 5 + CALLS.clear() + FLAKY["fail"] = False + summary = job.run(orchestrator=ORCHESTRATORS[orchestrator]()) + assert CALLS == ["flaky"] + assert (summary.nodes["node"].computed, summary.nodes["node"].reused) == (1, 5) + assert job.nodes["node"].as_table().num_rows == 6 + CALLS.clear() + summary = job.run(orchestrator=ORCHESTRATORS[orchestrator]()) + assert CALLS == [] + assert summary.nodes["node"].held == 6 + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_an_unchanged_rerun_touches_no_store(orchestrator: str) -> None: + db = InMemoryArrowDatabase() + job = _grouped_job(ROWS, db) + job.run(orchestrator=ORCHESTRATORS[orchestrator]()) + _add_pending_write(db) + before = _pending(db) + with mock.patch.object( + InMemoryArrowDatabase, "flush", side_effect=AssertionError("flushed") + ): + job.run(orchestrator=ORCHESTRATORS[orchestrator]()) + assert _pending(db) == before + + +def test_a_held_load_only_node_keeps_its_missing_inputs_in_the_summary() -> None: + db = InMemoryArrowDatabase() + _grouped_job(ROWS[:3], db).run() + job = _grouped_job(ROWS, db) + job.run(load_only=True) + summary = job.run(load_only=True) + rowwise = summary.nodes["rowwise"] + assert (rowwise.held, rowwise.not_stored) == (3, 3) + assert "held from the previous run (3 inputs without a stored result)" in repr( + summary + ) + + +class _Stop(BaseException): + """An interruption that is not an ``Exception``, as a cancellation is.""" + + +class StopsAtNodeEnd(NoOpObserver): + """An observer that interrupts the run at the end of one node.""" + + def __init__(self, label: str) -> None: + self.label = label + + def on_node_end(self, node_label: str, node_hash: str) -> None: + """Interrupt at the node's end.""" + if node_label == self.label: + raise _Stop() + + def contextualize(self, *identity_path: str) -> StopsAtNodeEnd: + """Keep this observer for every node.""" + return self + + +def test_an_interrupted_run_clears_the_nodes_it_ran() -> None: + job = _grouped_job(ROWS, InMemoryArrowDatabase()) + with pytest.raises(_Stop): + job.run(observer=StopsAtNodeEnd("summed")) + assert job.nodes["rowwise"].as_table().num_rows == 0 + assert job.nodes["rowwise"].is_stale + + +def test_a_loaded_operator_that_lost_an_input_names_it(tmp_path: Path) -> None: + tap = side_effect_pod(_deliver_x) + assert isinstance(tap, SideEffectPod) + db = InMemoryArrowDatabase() + job = PipelineJob(name="m2", database=db) + with job: + tapped: Any = tap.process(_source(), label="delivered") + Join()( + tapped, + DictSource( + [{"g": r["g"], "k": r["k"], "w": r["x"]} for r in ROWS], + tag_columns=["g", "k"], + source_id="w", + ), + label="joined", + ) + job.run() + job.save(tmp_path / "job.json") + loaded = PipelineJob.load(tmp_path / "job.json", database=db) + loaded.bind(sources={"s": _source()}) + with pytest.raises(RuntimeError, match=r"'joined'.*ITL-1094"): + loaded.run(load_only=True) + + +def test_an_async_cache_only_node_holds_the_rows_it_sent() -> None: + from orcapod.channels import Channel + from orcapod.core.nodes import FunctionJobNode + from orcapod.pipeline.serialization import LoadStatus + + table = pa.table( + {"k": [0, 1], "x": [1, 2]}, + schema=pa.schema( + [ + pa.field("k", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] + ), + ) + db = InMemoryArrowDatabase() + + def node() -> Any: + return FunctionJobNode( + inc.pod, # pyright: ignore[reportArgumentType] # the decorator's pod type is not declared as FunctionPodProtocol + ArrowTableStream(table, tag_columns=["k"]), + pipeline_database=db, + ) + + node().run() + cached = node() + # A node whose input is unavailable (as from_descriptor makes it) is the case here. + cached._load_status = LoadStatus.CACHE_ONLY # pyright: ignore[reportPrivateUsage] # the load status is what this test is about + + async def run() -> list[Any]: + inputs: Channel[Any] = Channel(buffer_size=4) + output: Channel[Any] = Channel(buffer_size=4) + await inputs.writer.close() + # The channel entry point is what this test is about. + await cached.async_execute(inputs.reader, output.writer) + return await output.reader.collect() + + sent = asyncio.run(run()) + assert len(sent) == 2 + assert cached.as_table().num_rows == 2 diff --git a/tests/test_pipeline/test_job_run_state_rule.py b/tests/test_pipeline/test_job_run_state_rule.py index 32761f012..ff58020fd 100644 --- a/tests/test_pipeline/test_job_run_state_rule.py +++ b/tests/test_pipeline/test_job_run_state_rule.py @@ -52,11 +52,13 @@ def _deliver(y: int, ctx: Any) -> None: @pytest.fixture(autouse=True) def _reset() -> None: + """Start every test with no recorded calls or deliveries.""" CALLS.clear() DELIVERED.clear() def _source(data: list[dict[str, int]]) -> DictSource: + """An in-memory source of ``data`` with tags ``g`` and ``k``.""" return DictSource(data, tag_columns=["g", "k"], source_id="s") @@ -75,10 +77,12 @@ def _build(data: list[dict[str, int]], db: Any) -> PipelineJob: def _run(job: PipelineJob, orchestrator: str, **kwargs: Any) -> Any: + """Run ``job`` on the named orchestrator.""" return job.run(orchestrator=ORCHESTRATORS[orchestrator](), **kwargs) def _totals(job: PipelineJob) -> list[tuple[int, int]]: + """The ``(g, total)`` rows of ``summed``, sorted.""" return sorted( (r["g"], r["total"]) for r in job.nodes["summed"].as_table().to_pylist() ) @@ -86,6 +90,7 @@ def _totals(job: PipelineJob) -> list[tuple[int, int]]: @pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) def test_a_second_run_of_an_unchanged_job_does_nothing(orchestrator: str) -> None: + """No computation, store access, source read or delivery; every node held.""" job = _build(ROWS, InMemoryArrowDatabase()) _run(job, orchestrator) assert len(DELIVERED) == 6 @@ -134,6 +139,7 @@ def test_a_cleared_node_runs_again_with_the_nodes_below_it(orchestrator: str) -> @pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) def test_clearing_a_side_effect_node_delivers_it_again(orchestrator: str) -> None: + """``clear_cache()`` on a side-effect node makes the next run deliver again.""" job = _build(ROWS, InMemoryArrowDatabase()) _run(job, orchestrator) DELIVERED.clear() @@ -145,6 +151,7 @@ def test_clearing_a_side_effect_node_delivers_it_again(orchestrator: str) -> Non @pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) def test_a_rebound_source_runs_every_node_below_it(orchestrator: str) -> None: + """Binding a new source object makes every node below it run; new inputs compute.""" job = _build(ROWS[:3], InMemoryArrowDatabase()) _run(job, orchestrator) CALLS.clear() @@ -158,6 +165,7 @@ def test_a_rebound_source_runs_every_node_below_it(orchestrator: str) -> None: def test_a_full_run_after_a_load_only_run_runs_the_load_only_nodes( orchestrator: str, ) -> None: + """Load-only rows are never current for a full run: it computes what was skipped.""" db = InMemoryArrowDatabase() _run(_build(ROWS[:3], db), orchestrator) CALLS.clear() @@ -173,6 +181,7 @@ def test_a_full_run_after_a_load_only_run_runs_the_load_only_nodes( @pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) def test_a_load_only_run_after_a_full_run_does_nothing(orchestrator: str) -> None: + """A full run's rows are current for a load-only run too.""" job = _build(ROWS, InMemoryArrowDatabase()) _run(job, orchestrator) CALLS.clear() @@ -182,6 +191,7 @@ def test_a_load_only_run_after_a_full_run_does_nothing(orchestrator: str) -> Non def test_run_async_follows_the_rule() -> None: + """``run_async`` holds current nodes as ``run`` does.""" job = _build(ROWS, InMemoryArrowDatabase()) asyncio.run(job.run_async()) CALLS.clear() @@ -193,11 +203,13 @@ def test_run_async_follows_the_rule() -> None: @pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) def test_a_failed_commit_clears_only_the_nodes_that_ran(orchestrator: str) -> None: + """A failed end-of-run commit clears the nodes that ran; held nodes keep rows.""" job = _build(ROWS, InMemoryArrowDatabase()) _run(job, orchestrator) job.nodes["summed"].clear_cache() def fail(self: Any) -> None: + """A flush that always fails.""" raise OSError("disk full") with ( From fd812c1326994588221f251e754d4ceb0b47ab51 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 03:13:24 -0800 Subject: [PATCH 33/63] test(pipeline): #379's async error-policy tests read a failed run's nodes as empty (ITL-1095) After a fail_fast raise every node the run ran reads as not run (empty, stale) under the current-run rule; the stored rows are checked through as_source(), sync and async alike. Co-Authored-By: Claude Opus 5.5 --- .../test_pipeline/test_async_error_policy.py | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/tests/test_pipeline/test_async_error_policy.py b/tests/test_pipeline/test_async_error_policy.py index 5d8cb2d13..e05ea7952 100644 --- a/tests/test_pipeline/test_async_error_policy.py +++ b/tests/test_pipeline/test_async_error_policy.py @@ -202,8 +202,9 @@ def ks(job: PipelineJob, label: str) -> list[str]: def stored(store: Any, build: Callable[[Any, list[int]], PipelineJob], label: str): - """The rows a fresh job over ``store`` reads for ``label`` without running.""" - return build(store, xs_failing_at(0)).nodes[label].as_table() + """Every row ``store`` holds for ``label``: the node's stored view (``as_source()``; + a node itself shows only its current run, ITL-1095).""" + return build(store, xs_failing_at(0)).nodes[label].as_source().as_table() @pytest.fixture(params=list(ORCHESTRATORS)) @@ -355,10 +356,10 @@ def test_fail_fast_leaves_the_nodes_with_no_run( ) -> None: """After a fail_fast raise, the failing node and the node after it hold no run. - A run that raised defines no rows (ITL-1093): each node reads as a node with no run, - every row stored under its content hash, also the downstream node the async run - cancelled. Another job stores rows of other values first, so a node still holding - its previous run's rows reads fewer rows than a node with no run. + A run that raised defines no rows (ITL-1093): each node it ran reads as a node with no + run, which is empty (ITL-1095), also the downstream node the async run cancelled; what + was stored stays readable through ``as_source()``. Another job stores rows of other + values first, so the stored view holds both jobs' rows. """ store = InMemoryArrowDatabase() run_job(build(store, [100 + i for i in range(N_ROWS)]), orchestrator, "continue") @@ -372,10 +373,11 @@ def test_fail_fast_leaves_the_nodes_with_no_run( run_job(job, orchestrator, "fail_fast") for label in ("f", "down"): - no_run = rows_of(build(store, xs).nodes[label].as_table()) - assert rows_of(job.nodes[label].as_table()) == no_run, label - # Also the other job's rows: not the previous run's rows only. - assert len(no_run) == 2 * N_ROWS - 1, label + node = job.nodes[label] + assert node.as_table().num_rows == 0, label + assert node.is_stale, label + # Every stored row stays: the other job's rows and this job's. + assert len(rows_of(node.as_source().as_table())) == 2 * N_ROWS - 1, label # The row ``sibling`` leaves uncomputed until the run that is cancelled. @@ -407,7 +409,8 @@ def test_a_node_cancelled_on_another_branch_holds_no_run() -> None: """A node the async run cancels, not downstream of the failure, holds no run either. ``g`` waits at its one uncomputed row until the failure of ``f`` cancels it. Its - previous run's rows are not held: it reads as a node with no run (ITL-1093). + previous run's rows are not held: it reads as a node with no run (ITL-1093), which is + empty (ITL-1095); its stored rows stay readable through ``as_source()``. """ store = InMemoryArrowDatabase() other = [100 + i for i in range(N_ROWS)] @@ -425,9 +428,10 @@ def test_a_node_cancelled_on_another_branch_holds_no_run() -> None: finally: sibling_mode["mode"] = "fail" - no_run = rows_of(branches_job(store, xs).nodes["g"].as_table()) - assert rows_of(job.nodes["g"].as_table()) == no_run - assert len(no_run) == 2 * N_ROWS - 1 + g = job.nodes["g"] + assert g.as_table().num_rows == 0 + assert g.is_stale + assert len(rows_of(g.as_source().as_table())) == 2 * N_ROWS - 1 def test_an_observer_failing_at_node_end_is_called_once(orchestrator: str) -> None: From f02d72ae4dfb3432599269dd1cdb8aeb5f05844b Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 03:29:43 -0800 Subject: [PATCH 34/63] fix(side_effects): a side-effect node filled by a load-only run shows loaded_only (ITL-1095) Every node kind a load-only run fills (function, operator, side-effect) reports ViewStatus.LOADED_ONLY; a source node is read as in any run and shows rows. The side-effect node's async load-only branch now comes before it reads its pod's configuration (a load-only run delivers nothing), so a podless node can reuse it. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/streams/view.py | 6 +++++ src/orcapod/side_effects.py | 25 +++++++++++++------ .../test_current_run_review_fixes.py | 14 +++++++++++ .../test_pipeline/test_job_run_state_rule.py | 4 +-- 4 files changed, 40 insertions(+), 9 deletions(-) diff --git a/src/orcapod/core/streams/view.py b/src/orcapod/core/streams/view.py index 8737372c9..defaad41f 100644 --- a/src/orcapod/core/streams/view.py +++ b/src/orcapod/core/streams/view.py @@ -83,6 +83,12 @@ class ViewStatus(str, Enum): "loaded only: the output of a load-only run, whose inputs may lack rows" ) +# A side-effect node whose rows came from a load-only run: it passed them on and delivered +# nothing (ITL-1095). +LOADED_ONLY_SIDE_EFFECT_NOTE = ( + "loaded only: a load-only run passed these rows on and delivered nothing" +) + _DEFAULT_NOTES = { ViewStatus.NOT_COMPUTED: NOT_COMPUTED_NOTE, ViewStatus.NOTHING_STORED: NOTHING_STORED_NOTE, diff --git a/src/orcapod/side_effects.py b/src/orcapod/side_effects.py index eda680270..bb7af8e10 100644 --- a/src/orcapod/side_effects.py +++ b/src/orcapod/side_effects.py @@ -26,6 +26,7 @@ schema_requires_io, ) from orcapod.core.streams.view import ( + LOADED_ONLY_SIDE_EFFECT_NOTE, SIDE_EFFECTS_NOTE, ViewStatus, StreamView, @@ -1191,13 +1192,23 @@ def head( return self._head_without_rows( config, ViewStatus.ROWS, note="its last run passed on no rows" ) - return StreamView.of_rows( + view = StreamView.of_rows( self, self.as_table(columns=config).slice(0, n), tag_columns=self.keys(columns=config)[0], total_rows=len(rows), origin="held", ) + return self._with_run_status(view) + + def _with_run_status(self, view: StreamView) -> StreamView: + """``view`` with the status of the node's last run: ``loaded_only`` after a + load-only run, which passed the rows on and delivered nothing (ITL-1095).""" + if not self._run_load_only: + return view + return dataclasses.replace( + view, status=ViewStatus.LOADED_ONLY, note=LOADED_ONLY_SIDE_EFFECT_NOTE + ) @override def run(self, *, load_only: bool = False) -> StreamView: @@ -1235,7 +1246,7 @@ def run(self, *, load_only: bool = False) -> StreamView: run_alone=lambda: self._run_alone(load_only=load_only), caller=sys._getframe(1), # pyright: ignore[reportPrivateUsage] # the caller's line is where a failure warning is shown ) - return self._view_of_rows(self.as_table(), "held") + return self._with_run_status(self._view_of_rows(self.as_table(), "held")) def _run_alone(self, *, load_only: bool) -> None: """Run this node over an input stream that is not a job node; commit its log.""" @@ -1303,12 +1314,8 @@ async def async_execute( f"SideEffectJobNode.async_execute expects exactly 1 input channel; " f"got {len(inputs)}." ) - max_concurrency = self._pod.pod_config.max_concurrency - sem = ( - asyncio.Semaphore(max_concurrency) if max_concurrency is not None else None - ) - self._held_rows = None + # A load-only run reads no pod configuration: it delivers nothing. if load_only: try: rows: list[tuple[TagProtocol, DataProtocol]] = [] @@ -1323,6 +1330,10 @@ async def async_execute( finally: await self._close_output(output) return + max_concurrency = self._pod.pod_config.max_concurrency + sem = ( + asyncio.Semaphore(max_concurrency) if max_concurrency is not None else None + ) # This run's rows, in the order they were sent downstream. passed_on: list[tuple[TagProtocol, DataProtocol]] = [] # Makes recording a row and sending it one step: a send blocked by a full channel diff --git a/tests/test_pipeline/test_current_run_review_fixes.py b/tests/test_pipeline/test_current_run_review_fixes.py index 8c950bb57..b4cd0c7f7 100644 --- a/tests/test_pipeline/test_current_run_review_fixes.py +++ b/tests/test_pipeline/test_current_run_review_fixes.py @@ -647,3 +647,17 @@ async def run() -> list[Any]: sent = asyncio.run(run()) assert len(sent) == 2 assert cached.as_table().num_rows == 2 + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_every_node_a_load_only_run_fills_says_so(orchestrator: str) -> None: + """Function, operator and side-effect nodes filled by a load-only run show + ``loaded_only``; a source node is read as in any run and shows its rows.""" + db = InMemoryArrowDatabase() + _tap_job(db).run() + job = _tap_job(db) + job.run(orchestrator=ORCHESTRATORS[orchestrator](), load_only=True) + for label in ("rows", "tap", "down"): + assert job.nodes[label].head().status is ViewStatus.LOADED_ONLY, label + assert job.nodes["tap"].run(load_only=True).status is ViewStatus.LOADED_ONLY + assert job.nodes["s"].head().status is ViewStatus.ROWS diff --git a/tests/test_pipeline/test_job_run_state_rule.py b/tests/test_pipeline/test_job_run_state_rule.py index a5e7fab1e..a59a6ae5d 100644 --- a/tests/test_pipeline/test_job_run_state_rule.py +++ b/tests/test_pipeline/test_job_run_state_rule.py @@ -11,7 +11,7 @@ from __future__ import annotations import asyncio -from typing import Any +from typing import Any, Literal from unittest import mock import pytest @@ -225,7 +225,7 @@ def fail(self: Any) -> None: @pytest.mark.parametrize("error_policy", ["continue", "fail_fast"]) @pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) def test_a_partly_held_run_under_each_error_policy( - orchestrator: str, error_policy: str + orchestrator: str, error_policy: Literal["continue", "fail_fast"] ) -> None: """Held nodes feed the nodes that run, on both orchestrators and error policies.""" job = _build(ROWS, InMemoryArrowDatabase()) From 8314c934be06b0293e5a45d6fe453803844b4756 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 03:59:12 -0800 Subject: [PATCH 35/63] fix(pipeline): third review round for the current-run rule (ITL-1095) CodeRabbit review of f02d72ae, plan approved by the coordinator: - the async orchestrator drains the inputs of a held node, so a direct run_async(dag, held_rows=...) never blocks a producer (both error policies); - the loaded-job missing-input check (ITL-1094) covers only the nodes the run includes, and exclusion below an unbound source passes through a node the load did not restore; - a commit interrupted by a non-Exception clears the nodes the run ran; - a CACHE_ONLY node's as_table judges staleness as head and iter_data do; - pre-v0.3 tables are listed only for a run that runs a node; - docs and changelog: a current side-effect node is not delivered again by any run until cleared; a source holding the data it read when built needs a new source object, clear_cache() re-reads only a polling source; test docstrings. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1095.md | 4 +- docs/concepts/function-pods.md | 17 +- src/orcapod/core/nodes/function_node.py | 13 +- src/orcapod/pipeline/async_orchestrator.py | 12 +- src/orcapod/pipeline/job.py | 50 +++-- src/orcapod/pipeline/node_run.py | 4 + .../test_current_run_review_fixes.py | 199 +++++++++++++++++- 7 files changed, 265 insertions(+), 34 deletions(-) diff --git a/changelog.d/ITL-1095.md b/changelog.d/ITL-1095.md index c6ab32168..3d8413b61 100644 --- a/changelog.d/ITL-1095.md +++ b/changelog.d/ITL-1095.md @@ -4,8 +4,8 @@ - A function or operator job node that has not run in this process is stale (`is_stale` is `True`; it has no modified time), and a job node is stale also when any function or operator job node it depends on is. Binding a source to a job's source slot (`job.bind(sources=...)`) marks the slot modified, so every node fed by it is stale and empty until it runs again. - A loaded operator node no longer raises from `as_table()` or `iter_data()`: it is empty, like every node that has not run. Its stored view (`as_source()`, `get_all_records()`) raises instead when the job file does not record its run's content hash and the store holds nothing under the hash the loaded job computes (ITL-1043). - A job node's `run()` is state-driven and runs what the node depends on (developer, November 13, 2026; ITL-1095). A node that holds a completed run and is not stale runs nothing: `run()` returns the rows it holds. Otherwise every job node it depends on that holds no current run runs first, each once, as `job.run()` would run that part of the job, then the node; the run's writes are committed when it ends (before, `node.run()` read an upstream node's stored rows, or nothing, and left its last pipeline record and a `LOG` operator's output uncommitted). `run()` of a node whose function pod is a placeholder, or that was loaded without one, raises unless `load_only=True`; a `CACHE_ONLY` node's `run()` raises (its input is unavailable): read its stored results with `as_source()`. A job node's `run()` takes one keyword-only argument, `load_only`; every other stream's `run()` still takes none. -- A side-effect job node follows the same rule: it is stale before a run, and while it holds a current run its `run()` (or a downstream node's `run()`) delivers nothing; to deliver again without `track_completion`, call `clear_cache()` on it first (`job.run()` still delivers as before). A load-only run passes its rows on and delivers nothing. -- `job.run()` and `job.run_async()` follow the same rule as `node.run()` (developer, November 14, 2026): a node that holds a current run is not run again, and its rows feed the nodes after it. Running an unchanged job a second time now computes nothing, reads and writes nothing in the store, does not read its sources and delivers no side effect, even without `track_completion` (before, every `job.run()` re-read every source and re-processed every node, and delivered every untracked side effect again). Nodes run when they have not run in this process, were cleared, are below a cleared node or a node that ran again, or below a source bound again or whose own modified time moved. After this change a re-run does not notice new data in a source whose object is unchanged (a file or table changed in place, a polling source with new rows): bind a new source object, or call `clear_cache()` on the first node below it, to read it again (a start-of-run check for new data is ITL-1102). The summary reports a node that was not run as held: "rowwise: 6 rows: held from the previous run" (`NodeRunSummary.held`). If the commit at the end of a run fails, the nodes it ran are cleared and the nodes it held keep their rows. +- A side-effect job node follows the same rule: it is stale before a run, and while it holds a current run neither its `run()`, nor a downstream node's `run()`, nor `job.run()` delivers anything; to deliver again without `track_completion`, call `clear_cache()` on it first. A load-only run passes its rows on and delivers nothing. +- `job.run()` and `job.run_async()` follow the same rule as `node.run()` (developer, November 14, 2026): a node that holds a current run is not run again, and its rows feed the nodes after it. Running an unchanged job a second time now computes nothing, reads and writes nothing in the store, does not read its sources and delivers no side effect, even without `track_completion` (before, every `job.run()` re-read every source and re-processed every node, and delivered every untracked side effect again). Nodes run when they have not run in this process, were cleared, are below a cleared node or a node that ran again, or below a source bound again or whose own modified time moved. After this change a re-run does not notice new data in a source whose object is unchanged (a file or table changed in place, a polling source with new rows; a start-of-run check for new data is ITL-1102). A source that holds the data it read when built (in-memory, CSV, Delta, database tables, a pinned `SpiralDBTableSource`) needs a new source object, bound with `job.bind(sources=...)`; for a `PollingSource`, `clear_cache()` on each node that reads it makes the next run poll it. The summary reports a node that was not run as held: "rowwise: 6 rows: held from the previous run" (`NodeRunSummary.held`). If the commit at the end of a run fails, the nodes it ran are cleared and the nodes it held keep their rows. - If a run raises or is interrupted, or the commit at the end of `job.run()` or `node.run()` fails, every node the run ran is cleared (not run, stale) before the error propagates; nodes it held keep their rows. A run that left failed rows (`error_policy="continue"`) is never current: the next run computes only the failed rows and reads the rest from the store. ### Added diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 98e1dc754..556ef5b56 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -610,13 +610,16 @@ A node is stale, and runs, when: - its source was bound again (`job.bind(sources=...)`), or the bound source's own modified time moved (a `CachedSource` that read new rows, a `PollingSource` that polled). -Staleness comes from modified times only. A source whose rows change while its object stays -the same (a CSV or Delta file rewritten in place, a database table that grew, a list a -`DictSource` was built from and that was changed afterwards, a polling source with new -rows) is not seen: a run holds the nodes below it. To read it again, build a new source -object and bind it (`job.bind(sources=...)`), which makes every node that reads it stale, -or call `clear_cache()` on each node that reads it directly (one per branch). (A start-of-run check for new data is planned, ITL-1102.) To deliver a -side effect again (with `track_completion=False`), call `clear_cache()` on its node first. +Staleness comes from modified times only, so a re-run does not notice new data in a source +whose object is unchanged (a start-of-run check for new data is planned, ITL-1102). Most +sources hold the data they read when they were built: an in-memory source (a `DictSource` +copies the list it is given), a CSV, Delta or database table source, and a +`SpiralDBTableSource` pinned to the table version it found. For those, build a new source +object and bind it (`job.bind(sources=...)`): every node that reads it becomes stale and +runs; clearing the nodes below would only read the same data again. A `PollingSource` polls +when it is read, so `clear_cache()` on each node that reads it directly (one per branch) +makes the next run poll it and process its new rows. To deliver a side effect again (with +`track_completion=False`), call `clear_cache()` on its node first. `node.last_run` holds the counts of a `node.run()`, as `job.last_run` does for `job.run()`, and a run's writes are committed when it ends; when everything is held there is nothing to diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index d332928d9..a9a6b57cf 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -618,7 +618,7 @@ def as_table( # hashes are then built from the same rows, and they are the current ones # (ITL-1045). ``iter_data`` clears a stale cache too, but only after the # cached table would already have been read. - if self.is_stale: + if self._read_finds_stale(): self.clear_cache() if self._cached_output_table is None: self._cached_output_table = self._table_from_pairs(list(self.iter_data())) @@ -637,6 +637,10 @@ def all_content_hashes() -> "pa.Array[Any]": self._cached_output_table, column_config, all_content_hashes ) + def _read_finds_stale(self) -> bool: + """Whether a read finds the rows this node holds out of date.""" + return self.is_stale + def _table_from_pairs( self, pairs: "list[tuple[TagProtocol, DataProtocol]]" ) -> "pa.Table": @@ -1629,6 +1633,13 @@ def _set_run_rows( self._cached_output_table = None self._cached_content_hash_column = None + def _read_finds_stale(self) -> bool: + """Whether a read finds the rows held out of date; never for ``CACHE_ONLY``, + whose input is unavailable (``head`` and ``iter_data`` judge it the same way).""" + from orcapod.pipeline.serialization import LoadStatus + + return self.load_status != LoadStatus.CACHE_ONLY and self.is_stale + def _holds_run(self, *, load_only: bool) -> bool: """Whether the node holds a completed run that a run need not redo. diff --git a/src/orcapod/pipeline/async_orchestrator.py b/src/orcapod/pipeline/async_orchestrator.py index 86d666d52..437b3ccd9 100644 --- a/src/orcapod/pipeline/async_orchestrator.py +++ b/src/orcapod/pipeline/async_orchestrator.py @@ -261,7 +261,11 @@ async def _run_async( writer = _CollectingWriter(writer, collector) if held_rows is not None and node in held_rows: - # Not run: the rows of its current run feed its successors. + # Not run: the rows of its current run feed its successors. Its + # own inputs are read and discarded, so no producer blocks on + # a channel nobody reads. + for upstream in in_edges.get(node, []): + tg.create_task(_drain(edge_readers[(upstream, node)])) _start_node( tg, failed_nodes, node, _send_held(held_rows[node], writer) ) @@ -634,6 +638,12 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: return NoOpLogger() +async def _drain(reader: Any) -> None: + """Read ``reader`` to its end, keeping nothing.""" + async for _ in reader: + pass + + async def _send_held( rows: list[tuple[TagProtocol, DataProtocol]], writer: WritableChannel[tuple[TagProtocol, DataProtocol]], diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index bd254eff2..231eaa795 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -1124,11 +1124,36 @@ def _prepare_run( "original pipeline definition." ) + # --- Build hash-keyed OrcaDAG for topological ordering + exclusion --- + hash_dag: OrcaDAG[str] = OrcaDAG() + for u, v in self._graph_edges: + hash_dag.add_edge(u, v) + for node_hash in self._persistent_node_map: + if node_hash not in hash_dag: + hash_dag.add_node(node_hash) + + # Walk topologically; exclude unbound sources and all their dependents. + excluded_hashes: set[str] = set() + for node_hash in hash_dag.topological_sort(): + if node_hash not in self._persistent_node_map: + # A node a loaded job did not restore (ITL-1094): below an unbound + # source it is left out too, so the nodes below it are as well. + if any(p in excluded_hashes for p in hash_dag.predecessors(node_hash)): + excluded_hashes.add(node_hash) + continue + node = self._persistent_node_map[node_hash] + if isinstance(node, SourceJobNode) and node.bound_source is None: + excluded_hashes.add(node_hash) + elif any(p in excluded_hashes for p in hash_dag.predecessors(node_hash)): + excluded_hashes.add(node_hash) + # A loaded job whose graph lost a node (PipelineJob.load does not restore # side-effect nodes yet, ITL-1094) cannot feed the nodes below it: every saved # input of a node must be a restored node. if self._loaded: for node_hash, node in self._persistent_node_map.items(): + if node_hash in excluded_hashes: + continue # below an unbound source: not part of this run if not isinstance(node, (FunctionJobNode, OperatorJobNode)): continue if node_hash not in self._hash_graph: @@ -1147,27 +1172,6 @@ def _prepare_run( "with node.as_source()." ) - # --- Build hash-keyed OrcaDAG for topological ordering + exclusion --- - hash_dag: OrcaDAG[str] = OrcaDAG() - for u, v in self._graph_edges: - hash_dag.add_edge(u, v) - for node_hash in self._persistent_node_map: - if node_hash not in hash_dag: - hash_dag.add_node(node_hash) - - # Walk topologically; exclude unbound sources and all their dependents. - excluded_hashes: set[str] = set() - for node_hash in hash_dag.topological_sort(): - if node_hash not in self._persistent_node_map: - continue - node = self._persistent_node_map[node_hash] - if isinstance(node, SourceJobNode) and node.bound_source is None: - excluded_hashes.add(node_hash) - elif any(p in excluded_hashes for p in hash_dag.predecessors(node_hash)): - excluded_hashes.add(node_hash) - - self._report_unclaimed_tables_written_before_v03(excluded_hashes) - # --- Build execution OrcaDAG from existing node objects (no cloning) --- exec_dag: OrcaDAG = OrcaDAG() for node_hash, node in self._persistent_node_map.items(): @@ -1207,6 +1211,10 @@ def _prepare_run( # records ahead of their results (ITL-919). # The state rule (ITL-1095): a node that holds a current run is not run again. plan = plan_run(exec_dag, load_only=load_only) + # Pre-v0.3 tables are reported for a run that runs something (ITL-954); a run + # that runs no node lists no store. + if plan.to_run: + self._report_unclaimed_tables_written_before_v03(excluded_hashes) # A load-only run writes nothing, and a run that runs no node touches no store, # so neither commits anything, not even pending writes of earlier operations. if table_committing is not None and not load_only and plan.to_run: diff --git a/src/orcapod/pipeline/node_run.py b/src/orcapod/pipeline/node_run.py index c79cda5ec..159578c94 100644 --- a/src/orcapod/pipeline/node_run.py +++ b/src/orcapod/pipeline/node_run.py @@ -195,6 +195,10 @@ def commit_run(stores: Sequence[ArrowDatabaseProtocol], ran: Sequence[Any]) -> N except Exception as exc: clear_run_rows(ran) raise StoreWriteError(store.base_path, flush=True) from exc + except BaseException: + # An interruption during the commit leaves its outcome unknown too. + clear_run_rows(ran) + raise def run_with_dependencies(target: Any, *, load_only: bool) -> RunSummary: diff --git a/tests/test_pipeline/test_current_run_review_fixes.py b/tests/test_pipeline/test_current_run_review_fixes.py index b4cd0c7f7..6f57ad949 100644 --- a/tests/test_pipeline/test_current_run_review_fixes.py +++ b/tests/test_pipeline/test_current_run_review_fixes.py @@ -11,7 +11,7 @@ import asyncio import json from pathlib import Path -from typing import Any +from typing import Any, Literal from unittest import mock import pyarrow as pa @@ -81,11 +81,13 @@ def _deliver(y: int, ctx: Any) -> None: @pytest.fixture(autouse=True) def _reset() -> None: + """Start every test with no recorded calls and a delivery that succeeds.""" CALLS.clear() FAIL["tap"] = False def _source(data: list[dict[str, int]] = ROWS) -> DictSource: + """An in-memory source of ``data`` with tags ``g`` and ``k``.""" return DictSource(data, tag_columns=["g", "k"], source_id="s") @@ -110,6 +112,7 @@ class FailsAtNodeEnd(NoOpObserver): """An observer whose ``on_node_end`` raises for one node.""" def __init__(self, label: str) -> None: + """Keep the label of the node this observer acts at.""" self.label = label def on_node_end(self, node_label: str, node_hash: str) -> None: @@ -128,6 +131,7 @@ def contextualize(self, *identity_path: str) -> FailsAtNodeEnd: def test_an_observer_failing_at_the_end_of_a_load_only_node_leaves_it_empty() -> None: + """An observer raising at a load-only node's end leaves the node empty and stale.""" db = InMemoryArrowDatabase() _grouped_job(ROWS, db).run() job = _grouped_job(ROWS, db) @@ -138,6 +142,7 @@ def test_an_observer_failing_at_the_end_of_a_load_only_node_leaves_it_empty() -> def test_a_failed_commit_of_a_node_run_alone_leaves_it_empty() -> None: + """A node run alone whose commit fails holds nothing afterwards.""" from orcapod.core.nodes import FunctionJobNode table = pa.table( @@ -156,6 +161,7 @@ def test_a_failed_commit_of_a_node_run_alone_leaves_it_empty() -> None: ) def fail(self: Any) -> None: + """A flush that always fails.""" raise OSError("disk full") with ( @@ -182,11 +188,13 @@ def test_a_failed_run_clears_every_node_it_ran() -> None: def test_a_failed_cascade_clears_every_node_it_ran() -> None: + """A node.run() cascade whose commit fails clears every node it ran.""" job = PipelineJob(name="f", database=InMemoryArrowDatabase()) with job: fails_on_21.pod(inc.pod(_source(), label="up"), label="down") def fail(self: Any) -> None: + """A flush that always fails.""" raise OSError("disk full") with ( @@ -200,6 +208,7 @@ def fail(self: Any) -> None: @pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) def test_a_failed_operator_store_leaves_the_operator_empty(orchestrator: str) -> None: + """An operator whose LOG write fails holds nothing afterwards.""" db = InMemoryArrowDatabase() job = PipelineJob(name="o", database=db) with job: @@ -230,6 +239,7 @@ def _tap_job(db: Any) -> PipelineJob: def test_a_failed_side_effect_run_makes_the_nodes_below_it_stale() -> None: + """A failed side-effect run makes it and every node below it stale.""" job = _tap_job(InMemoryArrowDatabase()) job.run() assert job.nodes["down"].as_table().num_rows == 6 @@ -243,6 +253,7 @@ def test_a_failed_side_effect_run_makes_the_nodes_below_it_stale() -> None: def test_a_side_effect_node_whose_output_cannot_close_holds_nothing() -> None: + """A side-effect run whose output close fails holds no rows.""" from orcapod.channels import Channel job = _tap_job(InMemoryArrowDatabase()) @@ -261,6 +272,7 @@ async def close(self) -> None: raise OSError("output cannot be closed") async def run() -> None: + """Run the node's channel entry point.""" inputs: Channel[Any] = Channel(buffer_size=16) for item in job.nodes["rows"].iter_data(): await inputs.writer.send(item) @@ -286,6 +298,7 @@ def _pending(db: InMemoryArrowDatabase) -> dict[str, int]: def _add_pending_write(db: InMemoryArrowDatabase) -> None: + """Leave one uncommitted write of another table in ``db``.""" db.add_records( ("elsewhere",), pa.table({"__rid": [b"r1"], "v": [1]}), @@ -297,6 +310,7 @@ def _add_pending_write(db: InMemoryArrowDatabase) -> None: def test_a_load_only_run_leaves_pending_writes_as_it_found_them( orchestrator: str, ) -> None: + """A load-only job run and cascade flush nothing: pending writes stay pending.""" db = InMemoryArrowDatabase() _grouped_job(ROWS, db).run() _add_pending_write(db) @@ -319,6 +333,7 @@ def test_a_load_only_run_leaves_pending_writes_as_it_found_them( def _source_table() -> ArrowTableSource: + """A three-row Arrow source with tag ``k`` and data ``x``, non-null.""" table = pa.table( {"k": [0, 1, 2], "x": [1, 2, 3]}, schema=pa.schema( @@ -332,6 +347,7 @@ def _source_table() -> ArrowTableSource: def test_a_standalone_log_run_commits_its_write(tmp_path: Path) -> None: + """A standalone LOG operator run commits its write before returning.""" node = OperatorJobNode( operator=SelectDataColumns(["x"]), input_streams=[_source_table()], @@ -350,6 +366,7 @@ def test_a_standalone_log_run_commits_its_write(tmp_path: Path) -> None: def test_a_standalone_log_run_after_a_load_only_run_stores(tmp_path: Path) -> None: + """A full standalone run after a load-only run computes and stores.""" db = InMemoryArrowDatabase() node = OperatorJobNode( operator=SelectDataColumns(["x"]), @@ -366,6 +383,7 @@ def test_a_standalone_log_run_after_a_load_only_run_stores(tmp_path: Path) -> No @pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) def test_an_operator_filled_by_a_load_only_run_says_so(orchestrator: str) -> None: + """An operator filled by a load-only run shows ``loaded_only``.""" db = InMemoryArrowDatabase() _grouped_job(ROWS, db).run() job = _grouped_job(ROWS, db) @@ -378,6 +396,7 @@ def test_an_operator_filled_by_a_load_only_run_says_so(orchestrator: str) -> Non def test_an_unrun_loaded_operator_without_its_operator_keeps_its_columns( tmp_path: Path, ) -> None: + """An operator node rebuilt without its operator keeps its saved columns.""" db = InMemoryArrowDatabase() job = PipelineJob(name="l", database=db) with job: @@ -405,6 +424,7 @@ def test_an_unrun_loaded_operator_without_its_operator_keeps_its_columns( def test_a_bound_source_with_no_modified_time_makes_its_slot_unknown() -> None: + """A source slot bound to a stream with no modified time has none.""" stream = ArrowTableStream(pa.table({"k": [0], "x": [1]}), tag_columns=["k"]) slot = SourceJobNode(name="s", bound_source=stream) assert slot.last_modified is not None @@ -435,6 +455,7 @@ def test_freshness_of_shared_ancestors_is_checked_once_per_traversal() -> None: calls = 0 def counting(self: Any) -> bool: + """Count the staleness checks, then check.""" nonlocal calls calls += 1 return original(self) @@ -457,6 +478,7 @@ def test_a_held_dependency_keeps_its_incomplete_inputs_in_the_summary() -> None: def test_a_run_with_every_node_held_says_so() -> None: + """The headline of a run with every node held says there was nothing to do.""" job = _grouped_job(ROWS, InMemoryArrowDatabase()) job.run() summary = job.run() @@ -475,6 +497,7 @@ def _deliver_x(x: int, ctx: Any) -> None: def test_a_loaded_job_without_its_side_effect_node_names_it(tmp_path: Path) -> None: + """A loaded job missing a side-effect node fails load-only naming ITL-1094.""" tap = side_effect_pod(_deliver_x) assert isinstance(tap, SideEffectPod) db = InMemoryArrowDatabase() @@ -531,6 +554,7 @@ def test_a_run_with_failed_rows_is_retried_by_the_next_run(orchestrator: str) -> @pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) def test_an_unchanged_rerun_touches_no_store(orchestrator: str) -> None: + """A re-run with nothing to run flushes nothing.""" db = InMemoryArrowDatabase() job = _grouped_job(ROWS, db) job.run(orchestrator=ORCHESTRATORS[orchestrator]()) @@ -544,6 +568,7 @@ def test_an_unchanged_rerun_touches_no_store(orchestrator: str) -> None: def test_a_held_load_only_node_keeps_its_missing_inputs_in_the_summary() -> None: + """A held load-only node still reports the inputs it skipped.""" db = InMemoryArrowDatabase() _grouped_job(ROWS[:3], db).run() job = _grouped_job(ROWS, db) @@ -564,6 +589,7 @@ class StopsAtNodeEnd(NoOpObserver): """An observer that interrupts the run at the end of one node.""" def __init__(self, label: str) -> None: + """Keep the label of the node this observer acts at.""" self.label = label def on_node_end(self, node_label: str, node_hash: str) -> None: @@ -577,6 +603,7 @@ def contextualize(self, *identity_path: str) -> StopsAtNodeEnd: def test_an_interrupted_run_clears_the_nodes_it_ran() -> None: + """A run interrupted by a non-Exception clears the nodes it ran.""" job = _grouped_job(ROWS, InMemoryArrowDatabase()) with pytest.raises(_Stop): job.run(observer=StopsAtNodeEnd("summed")) @@ -585,6 +612,7 @@ def test_an_interrupted_run_clears_the_nodes_it_ran() -> None: def test_a_loaded_operator_that_lost_an_input_names_it(tmp_path: Path) -> None: + """A loaded operator whose saved input was not restored names ITL-1094.""" tap = side_effect_pod(_deliver_x) assert isinstance(tap, SideEffectPod) db = InMemoryArrowDatabase() @@ -603,12 +631,22 @@ def test_a_loaded_operator_that_lost_an_input_names_it(tmp_path: Path) -> None: job.run() job.save(tmp_path / "job.json") loaded = PipelineJob.load(tmp_path / "job.json", database=db) - loaded.bind(sources={"s": _source()}) + loaded.bind( + sources={ + "s": _source(), + "w": DictSource( + [{"g": r["g"], "k": r["k"], "w": r["x"]} for r in ROWS], + tag_columns=["g", "k"], + source_id="w", + ), + } + ) with pytest.raises(RuntimeError, match=r"'joined'.*ITL-1094"): loaded.run(load_only=True) def test_an_async_cache_only_node_holds_the_rows_it_sent() -> None: + """An async CACHE_ONLY node holds the stored rows it sent.""" from orcapod.channels import Channel from orcapod.core.nodes import FunctionJobNode from orcapod.pipeline.serialization import LoadStatus @@ -625,6 +663,7 @@ def test_an_async_cache_only_node_holds_the_rows_it_sent() -> None: db = InMemoryArrowDatabase() def node() -> Any: + """A function node over the two-row table, on the shared store.""" return FunctionJobNode( inc.pod, # pyright: ignore[reportArgumentType] # the decorator's pod type is not declared as FunctionPodProtocol ArrowTableStream(table, tag_columns=["k"]), @@ -637,6 +676,7 @@ def node() -> Any: cached._load_status = LoadStatus.CACHE_ONLY # pyright: ignore[reportPrivateUsage] # the load status is what this test is about async def run() -> list[Any]: + """Run the node's channel entry point.""" inputs: Channel[Any] = Channel(buffer_size=4) output: Channel[Any] = Channel(buffer_size=4) await inputs.writer.close() @@ -661,3 +701,158 @@ def test_every_node_a_load_only_run_fills_says_so(orchestrator: str) -> None: assert job.nodes[label].head().status is ViewStatus.LOADED_ONLY, label assert job.nodes["tap"].run(load_only=True).status is ViewStatus.LOADED_ONLY assert job.nodes["s"].head().status is ViewStatus.ROWS + + +# --------------------------------------------------------------------------- +# Third review round +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("error_policy", ["continue", "fail_fast"]) +def test_a_direct_async_run_drains_the_inputs_of_a_held_node( + error_policy: Literal["continue", "fail_fast"], +) -> None: + """A held node's incoming channel is read even when the caller passes the whole + graph: a source with more rows than the channel buffer does not block the run.""" + from orcapod.pipeline.dag import OrcaDAG + + rows = [{"g": i % 2, "k": i, "x": i} for i in range(64)] + job = _grouped_job(rows, InMemoryArrowDatabase()) + job.run() + rowwise, grouped, summed = (job.nodes[n] for n in ("rowwise", "grouped", "summed")) + source = rowwise.upstreams[0] + dag: OrcaDAG[Any] = OrcaDAG() + for upstream, downstream in ( + (source, rowwise), + (rowwise, grouped), + (grouped, summed), + ): + dag.add_edge(upstream, downstream) + held = {rowwise: list(rowwise.iter_data())} + summed.clear_cache() + grouped.clear_cache() + orchestrator = AsyncPipelineOrchestrator(buffer_size=2, error_policy=error_policy) + + async def run() -> None: + """Run the whole graph with ``rowwise`` held, under a time limit.""" + await asyncio.wait_for(orchestrator.run_async(dag, held_rows=held), timeout=30) + + asyncio.run(run()) + assert summed.as_table().num_rows == 2 + + +def test_a_loaded_job_runs_its_bound_branch_beside_a_broken_unbound_one( + tmp_path: Path, +) -> None: + """The ITL-1094 check covers only the nodes the run includes: a branch below an + unbound source and a dropped node is left out, the bound branch loads.""" + tap = side_effect_pod(_deliver_x) + assert isinstance(tap, SideEffectPod) + db = InMemoryArrowDatabase() + job = PipelineJob(name="m3", database=db) + with job: + tapped: Any = tap.process(_source(), label="delivered") + inc.pod(tapped, label="broken") + dbl.pod( + inc.pod( + DictSource( + [{"g": 0, "k": 9, "x": 5}], + tag_columns=["g", "k"], + source_id="other", + ), + label="up2", + ), + label="bound_branch", + ) + job.run() + job.save(tmp_path / "job.json") + loaded = PipelineJob.load(tmp_path / "job.json", database=db) + loaded.bind( + sources={ + "other": DictSource( + [{"g": 0, "k": 9, "x": 5}], tag_columns=["g", "k"], source_id="other" + ) + } + ) + loaded.run(load_only=True) + assert loaded.nodes["bound_branch"].as_table().num_rows == 1 + assert loaded.nodes["broken"].as_table().num_rows == 0 + + +def test_a_cache_only_node_keeps_its_rows_over_an_input_with_no_modified_time() -> None: + """``as_table`` judges a ``CACHE_ONLY`` node as ``head`` and ``iter_data`` do: its + unavailable input does not make the rows it holds stale.""" + from orcapod.channels import Channel + from orcapod.core.nodes import FunctionJobNode + from orcapod.pipeline.serialization import LoadStatus + + table = pa.table( + {"k": [0, 1], "x": [1, 2]}, + schema=pa.schema( + [ + pa.field("k", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] + ), + ) + db = InMemoryArrowDatabase() + stream = ArrowTableStream(table, tag_columns=["k"]) + FunctionJobNode( + inc.pod, # pyright: ignore[reportArgumentType] # the decorator's pod type is not declared as FunctionPodProtocol + stream, + pipeline_database=db, + ).run() + cached = FunctionJobNode( + inc.pod, # pyright: ignore[reportArgumentType] # the decorator's pod type is not declared as FunctionPodProtocol + stream, + pipeline_database=db, + ) + # An unavailable input (as from_descriptor makes it) is the case here. + cached._load_status = LoadStatus.CACHE_ONLY # pyright: ignore[reportPrivateUsage] # the load status is what this test is about + + async def run() -> None: + """Feed nothing: a CACHE_ONLY node serves what is stored.""" + inputs: Channel[Any] = Channel(buffer_size=4) + output: Channel[Any] = Channel(buffer_size=4) + await inputs.writer.close() + await cached.async_execute(inputs.reader, output.writer) + _ = await output.reader.collect() + + asyncio.run(run()) + stream._set_modified_time(None) # pyright: ignore[reportPrivateUsage] # an input with no modified time is what this test is about + assert cached.as_table().num_rows == 2 + assert len(list(cached.iter_data())) == 2 + + +def test_a_commit_interrupted_by_a_non_exception_clears_the_nodes_it_ran() -> None: + """An interruption during the end-of-run commit (the last flush of the run) leaves + its outcome unknown: the nodes the run ran hold nothing.""" + original = InMemoryArrowDatabase.flush + calls = 0 + + def counting(self: Any) -> None: + """Count the flushes of a successful run.""" + nonlocal calls + calls += 1 + original(self) + + with mock.patch.object(InMemoryArrowDatabase, "flush", counting): + _grouped_job(ROWS, InMemoryArrowDatabase()).nodes["summed"].run() + total, calls = calls, 0 + + def interrupt_last(self: Any) -> None: + """Interrupt the final flush, as a ``KeyboardInterrupt`` would.""" + nonlocal calls + calls += 1 + if calls == total: + raise _Stop() + original(self) + + job = _grouped_job(ROWS, InMemoryArrowDatabase()) + with ( + mock.patch.object(InMemoryArrowDatabase, "flush", interrupt_last), + pytest.raises(_Stop), + ): + job.nodes["summed"].run() + for label in ("rowwise", "grouped", "summed"): + assert job.nodes[label].as_table().num_rows == 0, label From b22763fa54abb81b17de7e5512f4dc1220a85222 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 04:14:02 -0800 Subject: [PATCH 36/63] fix(pipeline): an interrupted end-of-run commit of job.run clears the nodes it ran (ITL-1095) CodeRabbit review of 8314c934 (outside diff, job.py:1318): a KeyboardInterrupt or cancellation during _finish_run's flush bypassed both handlers, so the nodes the run ran kept rows of a commit whose outcome is unknown. A BaseException there now clears plan.to_run and is re-raised unchanged, as commit_run already does for node.run(). Co-Authored-By: Claude Opus 5.5 --- src/orcapod/pipeline/job.py | 5 +++ .../test_current_run_review_fixes.py | 35 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 231eaa795..5669bf0d5 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -1328,6 +1328,11 @@ def _finish_run( flush_error.add_note(pending_note) clear_run_rows(plan.to_run) raise flush_error from exc + except BaseException: + # An interruption (KeyboardInterrupt, cancellation) leaves the commit's + # outcome unknown: no node holds the run's rows. Re-raised unchanged. + clear_run_rows(plan.to_run) + raise # Mutate self in place. self._has_run = True diff --git a/tests/test_pipeline/test_current_run_review_fixes.py b/tests/test_pipeline/test_current_run_review_fixes.py index 6f57ad949..429ccde6a 100644 --- a/tests/test_pipeline/test_current_run_review_fixes.py +++ b/tests/test_pipeline/test_current_run_review_fixes.py @@ -856,3 +856,38 @@ def interrupt_last(self: Any) -> None: job.nodes["summed"].run() for label in ("rowwise", "grouped", "summed"): assert job.nodes[label].as_table().num_rows == 0, label + + +def test_a_job_run_whose_final_commit_is_interrupted_clears_the_nodes_it_ran() -> None: + """An interruption during ``job.run()``'s end-of-run commit (its last flush) leaves + the outcome unknown: the nodes the run ran hold nothing, as for ``node.run()``.""" + original = InMemoryArrowDatabase.flush + calls = 0 + + def counting(self: Any) -> None: + """Count the flushes of a successful run.""" + nonlocal calls + calls += 1 + original(self) + + with mock.patch.object(InMemoryArrowDatabase, "flush", counting): + _grouped_job(ROWS, InMemoryArrowDatabase()).run() + total, calls = calls, 0 + + def interrupt_last(self: Any) -> None: + """Interrupt the final flush, as a ``KeyboardInterrupt`` would.""" + nonlocal calls + calls += 1 + if calls == total: + raise _Stop() + original(self) + + job = _grouped_job(ROWS, InMemoryArrowDatabase()) + with ( + mock.patch.object(InMemoryArrowDatabase, "flush", interrupt_last), + pytest.raises(_Stop), + ): + job.run() + for label in ("rowwise", "grouped", "summed"): + assert job.nodes[label].as_table().num_rows == 0, label + assert job.nodes[label].is_stale, label From 7de3512bdbe87d772a88c83ba0107f7cd8b0c58e Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 04:18:13 -0800 Subject: [PATCH 37/63] fix(pipeline): a run that leaves a node out still reports pre-v0.3 tables (ITL-1095) 8314c934 (CodeRabbit O2) listed pre-v0.3 tables only when the run ran a node. A run that runs nothing but leaves a node out below an unbound source must still report that node's tables in the job total, as the PR #324 test test_job_total_counts_the_tables_of_an_excluded_node guards. The listing is now skipped only for a run that neither runs nor leaves out any node. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/pipeline/job.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 5669bf0d5..c176d315b 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -1211,9 +1211,10 @@ def _prepare_run( # records ahead of their results (ITL-919). # The state rule (ITL-1095): a node that holds a current run is not run again. plan = plan_run(exec_dag, load_only=load_only) - # Pre-v0.3 tables are reported for a run that runs something (ITL-954); a run - # that runs no node lists no store. - if plan.to_run: + # Pre-v0.3 tables are reported for a run that runs a node or leaves one out + # (ITL-954): an excluded node reports nothing itself, so the job's total counts + # its tables. Only a run that runs nothing and leaves nothing out lists no store. + if plan.to_run or excluded_hashes: self._report_unclaimed_tables_written_before_v03(excluded_hashes) # A load-only run writes nothing, and a run that runs no node touches no store, # so neither commits anything, not even pending writes of earlier operations. From 86902ec0b16f0c09a4e5bed35634341c4e9728a5 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 05:36:24 -0800 Subject: [PATCH 38/63] chore(repro): reproduction scripts for the ITL-1121 review comments Co-Authored-By: Claude Opus 5.5 --- .../c1_observer_reads_in_on_node_end.py | 51 ++++++++++++++ .../itl_1121/c2_operator_lazy_failure.py | 45 ++++++++++++ .../itl_1121/c3_node_run_pending_writes.py | 61 ++++++++++++++++ .../itl_1121/c4_summary_side_effect.py | 54 +++++++++++++++ .../itl_1121/c5_load_only_execute_failure.py | 69 +++++++++++++++++++ .../c6_load_only_over_side_effect_stream.py | 57 +++++++++++++++ 6 files changed, 337 insertions(+) create mode 100644 superpowers/scripts/itl_1121/c1_observer_reads_in_on_node_end.py create mode 100644 superpowers/scripts/itl_1121/c2_operator_lazy_failure.py create mode 100644 superpowers/scripts/itl_1121/c3_node_run_pending_writes.py create mode 100644 superpowers/scripts/itl_1121/c4_summary_side_effect.py create mode 100644 superpowers/scripts/itl_1121/c5_load_only_execute_failure.py create mode 100644 superpowers/scripts/itl_1121/c6_load_only_over_side_effect_stream.py diff --git a/superpowers/scripts/itl_1121/c1_observer_reads_in_on_node_end.py b/superpowers/scripts/itl_1121/c1_observer_reads_in_on_node_end.py new file mode 100644 index 000000000..b74c8230b --- /dev/null +++ b/superpowers/scripts/itl_1121/c1_observer_reads_in_on_node_end.py @@ -0,0 +1,51 @@ +"""Comment 1: a function job node is stamped after on_node_end, so an observer that reads +the node in on_node_end finds it stale and clears it.""" + +from typing import Any + +from orcapod import PipelineJob, function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.pipeline.observer import NoOpObserver +from orcapod.sources import DictSource + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + return x + 1 + + +class Reader(NoOpObserver): + def __init__(self, job: Any) -> None: + self.job = job + self.seen: dict[str, int] = {} + + def contextualize(self, *identity_path: str) -> "Reader": + return self + + def on_node_end(self, node_label: str, node_hash: str) -> None: + node = self.job.nodes.get(node_label) + if node is not None: + self.seen[node_label] = node.as_table().num_rows + + +def build(db: Any) -> PipelineJob: + job = PipelineJob(name="j", database=db) + with job: + inc.pod(DictSource([{"k": i, "x": i} for i in range(3)], tag_columns=["k"], source_id="s"), label="rowwise") + return job + + +for name, orch in [("sync", SyncPipelineOrchestrator), ("async", AsyncPipelineOrchestrator)]: + for load_only in (False, True): + db = InMemoryArrowDatabase() + if load_only: + build(db).run() # store the results first + job = build(db) + obs = Reader(job) + job.run(orchestrator=orch(), observer=obs, load_only=load_only) + node = job.nodes["rowwise"] + print( + f"{name:5} load_only={load_only!s:5}: read in on_node_end={obs.seen.get('rowwise')}, " + f"after run: rows={node.as_table().num_rows} holds_current_run={node.holds_current_run(load_only=load_only)}" + ) diff --git a/superpowers/scripts/itl_1121/c2_operator_lazy_failure.py b/superpowers/scripts/itl_1121/c2_operator_lazy_failure.py new file mode 100644 index 000000000..a93767b7d --- /dev/null +++ b/superpowers/scripts/itl_1121/c2_operator_lazy_failure.py @@ -0,0 +1,45 @@ +"""Comment 2: an operator job node's run() over a lazy output (no database, or +CacheMode.OFF) materializes it in _run_view(), outside the try that clears the node.""" + +import pyarrow as pa + +from orcapod import function_pod +from orcapod.core.nodes.operator_node import OperatorJobNode +from orcapod.core.operators import MapData +from orcapod.core.streams import ArrowTableStream +from orcapod.databases import InMemoryArrowDatabase +from orcapod.types import CacheMode + +FAIL = {"on": True} + + +@function_pod(output_keys="y") +def flaky(x: int) -> int: + if FAIL["on"]: + raise RuntimeError("boom") + return x + 1 + + +schema = pa.schema([pa.field("id", pa.int64(), nullable=False), pa.field("x", pa.int64(), nullable=False)]) +base = ArrowTableStream(pa.table({"id": [1, 2], "x": [1, 2]}, schema=schema), tag_columns=["id"]) + +for label, db in [("no database", None), ("CacheMode.OFF", InMemoryArrowDatabase())]: + FAIL["on"] = True + upstream = flaky.pod(base) # a pod stream outside a job: lazy, raises when read + node = OperatorJobNode(operator=MapData({"y": "z"}), input_streams=(upstream,), pipeline_database=db, cache_mode=CacheMode.OFF) + try: + node.run() + except Exception as exc: + print(f"{label}: run() raised {type(exc).__name__}: {exc}") + print(f" after the failed run: holds_current_run={node.holds_current_run(load_only=False)} is_stale={node.is_stale} " + f"kept output installed={node._cached_output_stream is not None}") # private: what the comment is about + try: + print(f" as_table() rows={node.as_table().num_rows}") + except Exception as exc: + print(f" as_table() raised {type(exc).__name__}: {exc}") + print(f" head(): {node.head().status}") + FAIL["on"] = False + try: + print(f" second run() after the input recovers: rows={node.run().as_table().num_rows}") + except Exception as exc: + print(f" second run() raised {type(exc).__name__}: {exc}") diff --git a/superpowers/scripts/itl_1121/c3_node_run_pending_writes.py b/superpowers/scripts/itl_1121/c3_node_run_pending_writes.py new file mode 100644 index 000000000..c4da26214 --- /dev/null +++ b/superpowers/scripts/itl_1121/c3_node_run_pending_writes.py @@ -0,0 +1,61 @@ +"""Comment 3: node.run() starts without the pre-run flush that job.run() makes (ITL-919). + +The hazard ITL-919 guards against: a table-committing store reaches batch_size on a +table mid-run and commits pipeline records whose results are still pending. Check, on a +Delta store with a small batch_size and rows left pending before the run, that every +committed pipeline-record version only references committed results after node.run(). +""" + +import sys +import tempfile +from pathlib import Path + +import deltalake +import pyarrow as pa + +from orcapod import PipelineJob, function_pod +from orcapod.databases import DeltaTableDatabase +from orcapod.databases.storage_utils import delta_to_pyarrow_dataset +from orcapod.sources import DictSource + + +@function_pod(output_keys="y") +def double(x: int) -> int: + return 2 * x + + +def col(table: deltalake.DeltaTable, name: str) -> set: + return set(delta_to_pyarrow_dataset(table).to_table(columns=[name])[name].to_pylist()) + + +def check(root: Path) -> None: + tables = { + ("result" if "_result" in t.relative_to(root).parts else "pipeline"): t + for t in {log.parent for log in root.rglob("_delta_log")} + if t.name != "leftover" + } + result_history = deltalake.DeltaTable(str(tables["result"])).history() + pipeline_history = deltalake.DeltaTable(str(tables["pipeline"])).history() + print(f" pipeline commits: {len(pipeline_history)}, result commits: {len(result_history)}") + bad = 0 + for entry in pipeline_history: + records = deltalake.DeltaTable(str(tables["pipeline"]), version=entry["version"]) + versions = [r["version"] for r in result_history if r["timestamp"] <= entry["timestamp"]] + committed = col(deltalake.DeltaTable(str(tables["result"]), version=max(versions)), "__record_id") if versions else set() + if not col(records, "__data_id") <= committed: + bad += 1 + print(f" pipeline versions referencing uncommitted results: {bad}") + + +for batch_size, n in [(50, 120), (5, 40)]: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = DeltaTableDatabase(root, batch_size=batch_size) + for i in range(batch_size - 1): # pending, never flushed + store.add_record(("leftover",), f"left-{i}".encode(), pa.table({"v": pa.array([i], pa.int64())})) + job = PipelineJob(name="j", database=store) + with job: + double.pod(DictSource([{"k": i, "x": i} for i in range(n)], tag_columns=["k"], source_id="s"), label="dbl") + print(f"batch_size={batch_size}, rows={n}: node.run()") + job.nodes["dbl"].run() + check(root) diff --git a/superpowers/scripts/itl_1121/c4_summary_side_effect.py b/superpowers/scripts/itl_1121/c4_summary_side_effect.py new file mode 100644 index 000000000..980c76d2e --- /dev/null +++ b/superpowers/scripts/itl_1121/c4_summary_side_effect.py @@ -0,0 +1,54 @@ +"""Comment 4: function -> side-effect job; run, clear only the side-effect node, run again.""" + +from typing import Any + +from orcapod import PipelineJob, function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.side_effects import tap_pod +from orcapod.sources import DictSource + +DELIVERED: list[int] = [] + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + return x + 1 + + +def deliver(y: int, ctx: Any) -> None: + DELIVERED.append(y) + + +for name, orch in [("sync", SyncPipelineOrchestrator), ("async", AsyncPipelineOrchestrator)]: + for via in ("job.run", "node.run"): + DELIVERED.clear() + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + rows = inc.pod(DictSource([{"k": i, "x": i} for i in range(3)], tag_columns=["k"], source_id="s"), label="rowwise") + tap_pod(deliver).process(rows, label="tap") + job.run(orchestrator=orch()) + first = len(DELIVERED) + job.nodes["tap"].clear_cache() + DELIVERED.clear() + if via == "job.run": + summary = job.run(orchestrator=orch()) + else: + if name == "async": + continue # node.run() is sync only + job.nodes["tap"].run() + summary = job.nodes["tap"].last_run + print(f"--- {name} {via}: first run delivered {first}; second run delivered {len(DELIVERED)}") + print(repr(summary)) + +# The same with an operator node below the held function node instead of a side effect. +for name, orch in [("sync", SyncPipelineOrchestrator), ("async", AsyncPipelineOrchestrator)]: + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + rows = inc.pod(DictSource([{"k": i, "x": i} for i in range(3)], tag_columns=["k"], source_id="s"), label="rowwise") + rows.select_data_columns(["y"], label="op") + job.run(orchestrator=orch()) + job.nodes["op"].clear_cache() + summary = job.run(orchestrator=orch()) + print(f"--- {name} job.run, operator node cleared: op holds_current_run={job.nodes['op'].holds_current_run(load_only=False)}") + print(repr(summary)) diff --git a/superpowers/scripts/itl_1121/c5_load_only_execute_failure.py b/superpowers/scripts/itl_1121/c5_load_only_execute_failure.py new file mode 100644 index 000000000..55ea3f5d7 --- /dev/null +++ b/superpowers/scripts/itl_1121/c5_load_only_execute_failure.py @@ -0,0 +1,69 @@ +"""Comment 5: a sync load-only execute() whose input raises does not clear the node +(the async load-only branch and the full-run branch do).""" + +import asyncio +from typing import Any + +import pyarrow as pa + +from orcapod import PipelineJob, function_pod +from orcapod.channels import Channel +from orcapod.core.streams import ArrowTableStream +from orcapod.databases import InMemoryArrowDatabase +from orcapod.side_effects import tap_pod +from orcapod.sources import DictSource + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + return x + 1 + + +@function_pod(output_keys="z") +def boom(y: int) -> int: + raise RuntimeError("input failed") + + +def deliver(y: int, ctx: Any) -> None: + pass + + +def build() -> PipelineJob: + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + rows = inc.pod(DictSource([{"k": i, "x": i} for i in range(3)], tag_columns=["k"], source_id="s"), label="rowwise") + tapped: Any = tap_pod(deliver).process(rows, label="tap") + tapped.select_data_columns(["y"], label="down") + return job + + +schema = pa.schema([pa.field("k", pa.int64(), nullable=False), pa.field("y", pa.int64(), nullable=False)]) +failing = boom.pod(ArrowTableStream(pa.table({"k": [1], "y": [1]}, schema=schema), tag_columns=["k"])) + + +class FailingChannel: + def __aiter__(self): + return self + + async def __anext__(self): + raise RuntimeError("input failed") + + +for mode in ("sync full", "sync load_only", "async load_only"): + job = build() + job.run() + tap, down = job.nodes["tap"], job.nodes["down"] + try: + if mode == "sync full": + tap.execute(failing) + elif mode == "sync load_only": + tap.execute(failing, load_only=True) + else: + out = Channel() + asyncio.run(tap.async_execute([FailingChannel()], out.writer, load_only=True)) + except Exception as exc: + print(f"{mode}: raised {type(exc).__name__}: {exc}") + print( + f" tap: is_stale={tap.is_stale} rows={tap.as_table().num_rows}; " + f"down (ran before): is_stale={down.is_stale} rows={down.as_table().num_rows}" + ) diff --git a/superpowers/scripts/itl_1121/c6_load_only_over_side_effect_stream.py b/superpowers/scripts/itl_1121/c6_load_only_over_side_effect_stream.py new file mode 100644 index 000000000..89ffe095a --- /dev/null +++ b/superpowers/scripts/itl_1121/c6_load_only_over_side_effect_stream.py @@ -0,0 +1,57 @@ +"""Comment 6: a SideEffectJobNode over a standalone SideEffectPodStream, run with +load_only=True, delivers the upstream stream's side effects while reading it. + +Compared with the general case: a FunctionJobNode over a FunctionPodStream (a stream +outside a job), run with load_only=True, computes the upstream function while reading it. +""" + +from typing import Any + +import pyarrow as pa + +from orcapod import function_pod +from orcapod.core.nodes.function_node import FunctionJobNode +from orcapod.core.streams import ArrowTableStream +from orcapod.databases import InMemoryArrowDatabase +from orcapod.side_effects import SideEffectJobNode, tap_pod + +UP: list[int] = [] +OWN: list[int] = [] +CALLS: list[int] = [] + + +def deliver_up(x: int, ctx: Any) -> None: + UP.append(x) + + +def deliver_own(x: int, ctx: Any) -> None: + OWN.append(x) + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + CALLS.append(x) + return x + 1 + + +@function_pod(output_keys="z") +def dbl(y: int) -> int: + return 2 * y + + +schema = pa.schema([pa.field("k", pa.int64(), nullable=False), pa.field("x", pa.int64(), nullable=False)]) +base = ArrowTableStream(pa.table({"k": [1, 2, 3], "x": [1, 2, 3]}, schema=schema), tag_columns=["k"]) + +upstream: Any = tap_pod(deliver_up).process(base) # a SideEffectPodStream, outside a job +print("input type:", type(upstream).__name__) +node = SideEffectJobNode(tap_pod(deliver_own), upstream) +view = node.run(load_only=True) +print(f"side-effect node, run(load_only=True): upstream delivered {len(UP)}, own delivered {len(OWN)}, rows {view.as_table().num_rows}") +node.clear_cache() +node.run(load_only=True) +print(f" again after clear_cache(): upstream delivered {len(UP)} in total") + +fn_upstream: Any = inc.pod(base) # a FunctionPodStream, outside a job +fnode = FunctionJobNode(dbl.pod, fn_upstream, pipeline_database=InMemoryArrowDatabase()) +fview = fnode.run(load_only=True) +print(f"function node over a pod stream, run(load_only=True): upstream function called {len(CALLS)} times; status {fview.status}") From ef2bb44fd8723d6a7e63099a80e257385ab36b9e Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 05:38:05 -0800 Subject: [PATCH 39/63] docs(plan): ITL-1121 implementation plan Co-Authored-By: Claude Opus 5.5 --- ...6-11-14-itl-1121-node-view-review-fixes.md | 139 ++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md diff --git a/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md b/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md new file mode 100644 index 000000000..7adffc20c --- /dev/null +++ b/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md @@ -0,0 +1,139 @@ +# ITL-1121: job node view review fixes, implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan +> task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Fix the defects a full CodeRabbit review of PR #381 found in PR #380's code (the job +node current-run view, ITL-1095), in a PR stacked on #380, without new node-view behaviour and +without any hash, golden or stored-table change. + +**Architecture:** Each fix goes where the defect lives: the function node's run paths (stamp +before `on_node_end`), the operator node's `run()` (lazy materialization inside the clearing +`try`), `RunSummary` (headline from what the run plan ran), the side-effect node's sync +load-only branch (clear on failure), and one load-only guard in `pipeline/node_run.py` that +refuses an input which delivers when read. + +**Tech Stack:** Python 3.11+, pytest, `uv run`. + +Branch: `eywalker/itl-1121-job-node-view-itl-1095-defects-found-by-a`, based on PR #380 head +`7de3512b`. Draft PR against `main`, top line "Stacked on #380: merge after it. Fixes ITL-1121". + +Reproduction scripts: `superpowers/scripts/itl_1121/c1..c6_*.py` (committed). + +## Coordinator decisions (November 14, 2026) + +| # | Reproduced | Decision | +|---|---|---| +| 1 | yes, sync / async / load-only sync / load-only async | fix; observer test on all four paths + the node holds its rows after the run | +| 2 | yes (no DB, `CacheMode.OFF`) | fix | +| 3 | no (node.run commits per row; evidence c3) | no change; reply with evidence | +| 4 | yes, side-effect AND operator node cleared below a held function node | fix: `RunSummary(..., ran_nothing=...)`; tests: side-effect cleared, operator cleared, nothing ran, job with no function node; sync, async, node.run() | +| 5 | yes, through `execute()` only | fix (as async) | +| 6 | yes; general rule for outside streams | (a): a load-only run refuses, before reading, an input that would deliver (`SideEffectPodStream`), all three node kinds, direct input and cascade. A computing outside stream (FunctionPodStream) stays: write its reproduction + 3 lines for the coordinator | +| 7 | ITL-1097 | no fix; point to it | +| 8 | yes | docs: held node delivers nothing; `clear_cache()` first | +| 9 | - | recording function in `test-objective/integration/test_caching_flows.py` | + +## Files + +- Modify `src/orcapod/core/nodes/function_node.py`: `execute` (~2300-2322), `_execute_load_only` + (~1731-1733), `_async_execute_load_only` (~1786-1788), `async_execute` (~4003-4007); + `run_job_node` (~306-312, guard 6). +- Modify `src/orcapod/core/nodes/operator_node.py`: `run` (~1540-1580, fix 2 and guard 6). +- Modify `src/orcapod/pipeline/run_summary.py`: `RunSummary.__init__`, `_headline`. +- Modify `src/orcapod/pipeline/job.py`: `_finish_run` (`RunSummary(...)`). +- Modify `src/orcapod/pipeline/node_run.py`: `run_with_dependencies` (`RunSummary(...)`, guard 6), + new `refuse_delivering_inputs(node)`. +- Modify `src/orcapod/side_effects.py`: `SideEffectJobNode.execute` load-only branch; `run` uses + `run_job_node` (guard covered there). +- Modify `docs/concepts/function-pods.md` (~836-842). +- Modify `test-objective/integration/test_caching_flows.py` (`test_second_run_uses_cache`). +- Create `tests/test_pipeline/test_node_view_review_fixes.py` (all regression tests). +- Create `changelog.d/ITL-1121.md` (### Fixed), `design_issues/ITL-1121-NV1.md`. + +## Task 1: stamp before `on_node_end` (comment 1) + +- [ ] Test: observer subclassing `NoOpObserver` (contextualize returns self) reads + `job.nodes[label].as_table().num_rows` in `on_node_end`; parametrize orchestrator + sync/async and load_only False/True (load_only: a first job over the same store stores the + results). Assert read == 3 in the hook, and after the run `as_table().num_rows == 3` and + `holds_current_run(load_only=load_only)`. +- [ ] Run: fails (0 rows). +- [ ] Fix: in each of the four paths, call `self._update_modified_time()` right after + `self._set_run_rows(...)` and before `ctx_obs.on_node_end(...)`; delete the later stamp. A + raising observer still clears the node (the except branches call `clear_cache()`). +- [ ] Run the test and `tests/test_pipeline/test_node_staleness_after_run.py`, + `test_job_node_current_run_view.py`, `test_observer.py`: pass. Commit `fix(function_node): ...`. + +## Task 2: operator lazy materialization inside the clearing try (comment 2) + +- [ ] Test: `OperatorJobNode(MapData({"y": "z"}), (flaky.pod(base),), pipeline_database=db, + cache_mode=CacheMode.OFF)` for `db` None and `InMemoryArrowDatabase()`; the pod raises: + `run()` raises; then `holds_current_run(load_only=False) is False`, `is_stale`, and + `as_table().num_rows == 0`; after the pod recovers `run()` gives 2 rows. +- [ ] Fix: in `OperatorJobNode.run`, build the view inside the `try` for the branch that + computed (`return self._run_view(computed=computed)` moved into the `try` when + `computed`), keeping `except BaseException: self.clear_cache(); raise`. +- [ ] Run test + `tests/test_core/operators/test_operator_node*.py`. Commit. + +## Task 3: summary headline from what ran (comment 4) + +- [ ] Tests (sync, async; and `node.run()`): + - function -> tap; run; `tap.clear_cache()`; run: delivered 3 and the headline is not + "nothing to do" (`"Run of job 'j': all rows succeeded."`). + - function -> select (operator); `op.clear_cache()`; run: headline not "nothing to do". + - nothing ran: second run of an unchanged job says "nothing to do". + - a job with no function node (source -> select): a second run says "nothing to do"; the + first says "all rows succeeded". +- [ ] Fix: `RunSummary.__init__(..., ran_nothing: bool = False)` stored as `self.ran_nothing` + (documented in Attributes and Args); `_headline` returns the "nothing to do" line iff + `self.ran_nothing`. Callers: `job._finish_run` and `node_run.run_with_dependencies` pass + `ran_nothing=not plan.to_run`. +- [ ] Run test + `tests/test_pipeline/test_job_run_state_rule.py test_job_run_summary.py + test_node_run_cascade.py`. Commit. + +## Task 4: sync load-only execute clears on failure (comment 5) + +- [ ] Test: job `rowwise -> tap -> down`; `job.run()`; `tap.execute(failing, load_only=True)` + raises; `tap.is_stale` and `down.is_stale`, both read 0 rows. (`execute` is the node's + protocol entry point; this test is about it: one-line comment.) +- [ ] Fix: wrap the load-only branch in `try: ... except BaseException: self.clear_cache(); raise`. +- [ ] Commit. + +## Task 5: a load-only run refuses an input that delivers (comment 6) + +- [ ] Tests: standalone `SideEffectJobNode`, `FunctionJobNode`, `OperatorJobNode` over a + `SideEffectPodStream`: `run(load_only=True)` raises `ValueError` and nothing is delivered; + cascade: a job node over a standalone node over a `SideEffectPodStream`... (built as nodes + outside a job, `FunctionJobNode(pod, SideEffectJobNode(...))`, only if constructible; else + the direct cases plus the cascade through `run_with_dependencies` with a job node input). + Full runs are unchanged (deliver as before). +- [ ] Fix: `node_run.refuse_delivering_inputs(node)` raises `ValueError` naming the node and + the input when an upstream is a `SideEffectPodStream`. Called when `load_only` in + `run_job_node` (alone path, before `run_alone`), in `OperatorJobNode.run` (alone path), and in + `run_with_dependencies` for every node of `plan.to_run` before the orchestrator starts. +- [ ] Write c7 reproduction for FunctionPodStream computing in a load-only run (already in c6) + and three lines for the coordinator. +- [ ] Commit. + +## Task 6: docs (8), test-objective (9), changelog, design issue + +- [ ] `function-pods.md`: the sample comment `node.run() # holds a current run: delivers + nothing`; the bullet: with `track_completion=False`, a run delivers every row, but a node + that holds a current run is not run again: `clear_cache()` it first. Mention the load-only + refusal sentence in the load-only paragraph if one exists. +- [ ] `test_caching_flows.py`: recording `_double`, assert no call during `node2.run()`. +- [ ] `changelog.d/ITL-1121.md` (### Fixed), `design_issues/ITL-1121-NV1.md` (resolved). +- [ ] `uv run python scripts/assemble_changelog.py --check`, `scripts/design_issues.py --check`, + docs tests. Commit. + +## Task 7: finish + +- [ ] ruff format/check, basedpyright exit code, full `uv run pytest tests/` and + `test-objective/`. Push, draft PR, `orca linear attach`, `orca worktree set --pr`. +- [ ] Reply on the nine CodeRabbit threads/review-body findings on #381 (signed). +- [ ] CI green, CodeRabbit round on the new PR addressed; worker_done. + +## Progress notes + +- 2026-11-14: reproduction table sent and approved (see decisions). Scripts committed (86902ec0). From b62feae72e654fd1854ee71500b7ea370f02a45c Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 05:42:36 -0800 Subject: [PATCH 40/63] fix(function_node): stamp a node before on_node_end, so an observer reading it keeps its rows (ITL-1121) An observer that read a function job node in on_node_end found it with no modified time (stale) and cleared the run's rows; the node then read empty after the run. Sync, async and both load-only paths now stamp before the observer hears the end. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/function_node.py | 22 +++-- .../test_node_view_review_fixes.py | 87 +++++++++++++++++++ 2 files changed, 100 insertions(+), 9 deletions(-) create mode 100644 tests/test_pipeline/test_node_view_review_fixes.py diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index a9a6b57cf..f3339300c 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -1729,8 +1729,9 @@ def _execute_load_only( if entry[1] is not None: output.append((entry[0], entry[1])) self._set_run_rows(loaded_ids, load_only=True, not_stored=not_stored) - ctx_obs.on_node_end(node_label, node_hash) + # Current before the observer hears the end, so a read there sees the rows. self._update_modified_time() + ctx_obs.on_node_end(node_label, node_hash) except BaseException: # A run that raised (an observer at the node's end included) defines no rows. self.clear_cache() @@ -1784,8 +1785,9 @@ async def _async_execute_load_only( if entry[1] is not None: await output.send((entry[0], entry[1])) self._set_run_rows(loaded_ids, load_only=True, not_stored=not_stored) - ctx_obs.on_node_end(node_label, node_hash) + # Current before the observer hears the end, so a read there sees the rows. self._update_modified_time() + ctx_obs.on_node_end(node_label, node_hash) except BaseException: # A run that raised or was cancelled defines no rows. self.clear_cache() @@ -2307,6 +2309,10 @@ def execute( ) # The node's rows are those of this run's inputs, whatever else it caches. self._set_run_rows(emitted_entry_ids, failed=failed_rows) + # Mark this node as freshly computed before the observer hears the end, so + # a read there (and every later one) serves this run's rows instead of + # finding the node stale and clearing them (ITL-1121). + self._update_modified_time() ctx_obs.on_node_end(node_label, node_hash) except Exception as run_error: self._commit_after_failure(run_error) @@ -2317,9 +2323,6 @@ def execute( self.clear_cache() raise - # Mark this node as freshly computed so subsequent iter_data() calls - # skip the is_stale check and serve results directly from the in-memory cache. - self._update_modified_time() return output def _process_data_internal( @@ -3998,13 +4001,14 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: run_id=run_id, ) + # The run is complete: stamp the node as the sync execute() does, before + # the observer hears the end (a read there sees the rows, ITL-1121) and + # before the finally closes the output, so every downstream node, which + # stamps after it has seen that close, is newer than this one (ITL-1042). + self._update_modified_time() # Ended for the observer here, also when the call raises: not again below. node_open = False ctx_obs.on_node_end(node_label, node_hash) - # The run is complete: stamp the node as the sync execute() does, before - # the finally closes the output, so every downstream node, which stamps - # after it has seen that close, is newer than this one (ITL-1042). - self._update_modified_time() except asyncio.CancelledError as cancelled: # Another node's failure cancelled this one: commit what it wrote, as a # failing node does. The job's flush commits neither a node's ephemeral diff --git a/tests/test_pipeline/test_node_view_review_fixes.py b/tests/test_pipeline/test_node_view_review_fixes.py new file mode 100644 index 000000000..9b9ba95f3 --- /dev/null +++ b/tests/test_pipeline/test_node_view_review_fixes.py @@ -0,0 +1,87 @@ +"""Defects in the job node current-run view found by a full review (ITL-1121). + +The job node view (ITL-1095): a job node shows its current run; a run that raises leaves the +nodes it ran empty and stale; a node that holds a current run is not run again; a load-only +run computes, delivers and writes nothing. Each test reproduces one review comment through the +public interface (``superpowers/scripts/itl_1121`` holds the scripts). +""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.pipeline.observer import NoOpObserver +from orcapod.sources import DictSource + +ORCHESTRATORS = {"sync": SyncPipelineOrchestrator, "async": AsyncPipelineOrchestrator} + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + """Return ``x + 1``.""" + return x + 1 + + +def _source() -> DictSource: + """Three rows, tag ``k``, data ``x``.""" + return DictSource( + [{"k": i, "x": i} for i in range(3)], tag_columns=["k"], source_id="s" + ) + + +# --------------------------------------------------------------------------- +# Comment 1: a function node is stamped before on_node_end +# --------------------------------------------------------------------------- + + +class _ReadingObserver(NoOpObserver): + """Reads each job node in ``on_node_end``: the row count it sees, by label.""" + + def __init__(self, job: PipelineJob) -> None: + self.job = job + self.seen: dict[str, int] = {} + + def contextualize(self, *identity_path: str) -> _ReadingObserver: + """The same observer for every node.""" + return self + + def on_node_end(self, node_label: str, node_hash: str) -> None: + """Read the node that just ended.""" + node = self.job.nodes.get(node_label) + if node is not None: + self.seen[node_label] = node.as_table().num_rows + + +def _rowwise_job(db: Any) -> PipelineJob: + """``rowwise`` = ``inc`` over three rows.""" + job = PipelineJob(name="j", database=db) + with job: + inc.pod(_source(), label="rowwise") + return job + + +@pytest.mark.parametrize("load_only", [False, True]) +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_an_observer_reading_a_node_in_on_node_end_sees_its_rows( + orchestrator: str, load_only: bool +) -> None: + """The node is current when ``on_node_end`` fires, and keeps its rows after the run.""" + db = InMemoryArrowDatabase() + if load_only: + _rowwise_job(db).run() # store the results a load-only run loads + job = _rowwise_job(db) + observer = _ReadingObserver(job) + job.run( + orchestrator=ORCHESTRATORS[orchestrator](), + observer=observer, + load_only=load_only, + ) + node = job.nodes["rowwise"] + assert observer.seen["rowwise"] == 3 + assert node.as_table().num_rows == 3 + assert node.holds_current_run(load_only=load_only) From a4a95041185d3e7874ea08abb2d4a3261ffa126b Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 05:43:23 -0800 Subject: [PATCH 41/63] fix(operator_node): a run whose lazy output fails to compute holds no run (ITL-1121) Without a database, and in CacheMode.OFF, an operator node's run() computes its output while building the view, which was outside the try that clears a failed run: the node then claimed a current run and its reads raised. The view of a run that computed is now built inside it. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/operator_node.py | 4 ++ .../test_node_view_review_fixes.py | 58 +++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 10a55babe..e32c592a8 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -1573,6 +1573,10 @@ def run(self, *, load_only: bool = False) -> StreamView: ) self._update_modified_time() computed = True + if computed: + # Without a database and in OFF mode the output is lazy: building + # the view computes it, so a failure there is the run's (ITL-1121). + return self._run_view(computed=True) except BaseException: # A run that raised defines no rows. self.clear_cache() diff --git a/tests/test_pipeline/test_node_view_review_fixes.py b/tests/test_pipeline/test_node_view_review_fixes.py index 9b9ba95f3..662e5feb1 100644 --- a/tests/test_pipeline/test_node_view_review_fixes.py +++ b/tests/test_pipeline/test_node_view_review_fixes.py @@ -85,3 +85,61 @@ def test_an_observer_reading_a_node_in_on_node_end_sees_its_rows( assert observer.seen["rowwise"] == 3 assert node.as_table().num_rows == 3 assert node.holds_current_run(load_only=load_only) + + +# --------------------------------------------------------------------------- +# Comment 2: an operator node's lazy output that fails to materialize clears the node +# --------------------------------------------------------------------------- + +_FAIL = {"on": True} + + +@function_pod(output_keys="y") +def flaky(x: int) -> int: + """Raise while ``_FAIL["on"]``; else return ``x + 1``.""" + if _FAIL["on"]: + raise RuntimeError("input failed") + return x + 1 + + +def _base_stream() -> Any: + """Two rows, tag ``id`` and data ``x``, both non-nullable ``int64``.""" + import pyarrow as pa + + from orcapod.core.streams import ArrowTableStream + + schema = pa.schema( + [ + pa.field("id", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] + ) + return ArrowTableStream( + pa.table({"id": [1, 2], "x": [1, 2]}, schema=schema), tag_columns=["id"] + ) + + +@pytest.mark.parametrize("with_database", [False, True]) +def test_an_operator_run_whose_lazy_output_raises_holds_no_run( + with_database: bool, +) -> None: + """No database, or ``CacheMode.OFF``: the output is computed when the view is built; + if that raises, the node reads as one with no run, and the next run computes.""" + from orcapod.core.nodes.operator_node import OperatorJobNode + from orcapod.core.operators import MapData + from orcapod.types import CacheMode + + _FAIL["on"] = True + node = OperatorJobNode( + operator=MapData({"y": "z"}), + input_streams=(flaky.pod(_base_stream()),), + pipeline_database=InMemoryArrowDatabase() if with_database else None, + cache_mode=CacheMode.OFF, + ) + with pytest.raises(RuntimeError, match="input failed"): + node.run() + assert not node.holds_current_run(load_only=False) + assert node.is_stale + assert node.as_table().num_rows == 0 + _FAIL["on"] = False + assert node.run().as_table().num_rows == 2 From 3404b7efbf310b7ba1d65a65c7b983e760778a6e Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 05:44:37 -0800 Subject: [PATCH 42/63] fix(run_summary): say 'nothing to do' only when the run ran no node (ITL-1121) The headline judged only the function nodes' counts, so a run that ran a side-effect node (and delivered) or an operator node below a held function node said 'nothing to do; every node holds a current run'. RunSummary takes ran_nothing from the run's plan (job.run, run_async and node.run). Co-Authored-By: Claude Opus 5.5 --- src/orcapod/pipeline/job.py | 1 + src/orcapod/pipeline/node_run.py | 1 + src/orcapod/pipeline/run_summary.py | 13 ++- .../test_node_view_review_fixes.py | 81 +++++++++++++++++++ 4 files changed, 92 insertions(+), 4 deletions(-) diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index c176d315b..ad1f0d068 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -1343,6 +1343,7 @@ def _finish_run( "/".join(self._name), run_node_summaries(prepared.exec_dag, counter, held=plan.held_counts), load_only=load_only, + ran_nothing=not plan.to_run, ) return self._last_run diff --git a/src/orcapod/pipeline/node_run.py b/src/orcapod/pipeline/node_run.py index 159578c94..bdeb0b507 100644 --- a/src/orcapod/pipeline/node_run.py +++ b/src/orcapod/pipeline/node_run.py @@ -270,4 +270,5 @@ def visit(node: Any) -> None: run_node_summaries(graph, counter, held=plan.held_counts), load_only=load_only, target=target.label, + ran_nothing=not plan.to_run, ) diff --git a/src/orcapod/pipeline/run_summary.py b/src/orcapod/pipeline/run_summary.py index e3bc3dac2..e59e571f0 100644 --- a/src/orcapod/pipeline/run_summary.py +++ b/src/orcapod/pipeline/run_summary.py @@ -76,6 +76,9 @@ class RunSummary: it computed nothing; ``reused`` counts the rows it loaded. target: For the run of one node and the nodes it depends on (``node.run()``), that node's label; ``None`` for a job run. + ran_nothing: Whether the run ran no node: every node held a current run + (ITL-1095). Decided by the run's plan, which counts every kind of node, so a + run that ran only a side-effect or operator node is not "nothing to do". """ def __init__( @@ -85,6 +88,7 @@ def __init__( *, load_only: bool = False, target: str | None = None, + ran_nothing: bool = False, ) -> None: """Build the summary. @@ -93,11 +97,13 @@ def __init__( nodes: One entry per function node of the run, in run order. load_only: Whether the run loaded stored results only. target: The label of the node whose ``run()`` this was, if any. + ran_nothing: Whether the run ran no node (every node held a current run). """ self.job_name: str = job_name self.nodes: Mapping[str, NodeRunSummary] = {node.label: node for node in nodes} self.load_only: bool = load_only self.target: str | None = target + self.ran_nothing: bool = ran_nothing @property def failed_rows(self) -> int: @@ -152,10 +158,9 @@ def _headline(self) -> str: if self.target is None else f"node {self.target!r} and the nodes it depends on" ) - if self.nodes and all( - node.held and not (node.computed or node.reused or node.failed) - for node in self.nodes.values() - ): + # From what the run ran, not from the function nodes' counts: a run that ran + # only a side-effect or operator node did something (ITL-1121). + if self.ran_nothing: run = "Load-only run" if self.load_only else "Run" return f"{run} of {subject}: nothing to do; every node holds a current run." if self.load_only: diff --git a/tests/test_pipeline/test_node_view_review_fixes.py b/tests/test_pipeline/test_node_view_review_fixes.py index 662e5feb1..1e1470592 100644 --- a/tests/test_pipeline/test_node_view_review_fixes.py +++ b/tests/test_pipeline/test_node_view_review_fixes.py @@ -143,3 +143,84 @@ def test_an_operator_run_whose_lazy_output_raises_holds_no_run( assert node.as_table().num_rows == 0 _FAIL["on"] = False assert node.run().as_table().num_rows == 2 + + +# --------------------------------------------------------------------------- +# Comment 4: the summary says "nothing to do" only when the run ran nothing +# --------------------------------------------------------------------------- + +_DELIVERED: list[int] = [] +_NOTHING_TO_DO = "nothing to do; every node holds a current run" + + +def _deliver(y: int, ctx: Any) -> None: + """A side effect: record the delivered value.""" + _DELIVERED.append(y) + + +def _job_below_rowwise(below: str) -> PipelineJob: + """``rowwise`` (``inc``) with a side-effect node or an operator node ``below`` it.""" + from orcapod.side_effects import tap_pod + + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + rows = inc.pod(_source(), label="rowwise") + if below == "side_effect": + tap_pod(_deliver).process(rows, label="below") + else: + rows.select_data_columns(["y"], label="below") + return job + + +def _run_job_or_node(job: PipelineJob, how: str) -> Any: + """Run ``job`` on an orchestrator, or run its node ``below``; return the summary.""" + if how == "node.run": + job.nodes["below"].run() + return job.nodes["below"].last_run + return job.run(orchestrator=ORCHESTRATORS[how]()) + + +@pytest.mark.parametrize("how", ["sync", "async", "node.run"]) +@pytest.mark.parametrize("below", ["side_effect", "operator"]) +def test_a_run_that_ran_a_cleared_node_is_not_nothing_to_do( + below: str, how: str +) -> None: + """A node cleared below a held function node runs again (a side effect delivers); + the summary says what ran, not that there was nothing to do.""" + _DELIVERED.clear() + job = _job_below_rowwise(below) + _run_job_or_node(job, how) + job.nodes["below"].clear_cache() + _DELIVERED.clear() + summary = _run_job_or_node(job, how) + if below == "side_effect": + assert len(_DELIVERED) == 3 + assert job.nodes["below"].holds_current_run(load_only=False) + assert summary.nodes["rowwise"].held == 3 + assert _NOTHING_TO_DO not in repr(summary) + assert repr(summary).splitlines()[0].endswith("all rows succeeded.") + + +# A node.run() of a node that holds a current run returns at once and makes no summary +# (node.last_run stays the previous one), so "nothing ran" is a job run's case. +@pytest.mark.parametrize("how", ["sync", "async"]) +@pytest.mark.parametrize("below", ["side_effect", "operator"]) +def test_a_run_that_ran_nothing_is_nothing_to_do(below: str, how: str) -> None: + """A second run of an unchanged job runs nothing and says so.""" + job = _job_below_rowwise(below) + _run_job_or_node(job, how) + summary = _run_job_or_node(job, how) + assert _NOTHING_TO_DO in repr(summary).splitlines()[0] + + +@pytest.mark.parametrize("how", ["sync", "async", "node.run"]) +def test_a_job_with_no_function_node_says_what_ran(how: str) -> None: + """Only an operator: the first run succeeds; a second job run has nothing to do.""" + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _source().select_data_columns(["x"], label="below") + first = _run_job_or_node(job, how) + assert repr(first).splitlines()[0].endswith("all rows succeeded.") + if how != "node.run": # a held node.run() makes no summary (see above) + second = _run_job_or_node(job, how) + assert _NOTHING_TO_DO in repr(second).splitlines()[0] From 207f4ecc8c1b03fa6ac39fc0d8edebb6d4be8a27 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 05:45:27 -0800 Subject: [PATCH 43/63] fix(side_effects): a load-only execute whose input raises clears the node (ITL-1121) The sync load-only branch dropped its held rows but kept its modified time, so the node and the nodes below it were not stale after the failure. It now clears the node, as the async load-only branch and the delivering branches do. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/side_effects.py | 7 +- .../test_node_view_review_fixes.py | 77 +++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/src/orcapod/side_effects.py b/src/orcapod/side_effects.py index bb7af8e10..e9c9a7cf9 100644 --- a/src/orcapod/side_effects.py +++ b/src/orcapod/side_effects.py @@ -1083,7 +1083,12 @@ def execute( """ self._held_rows = None if load_only: - rows = list(input_stream.iter_data()) + try: + rows = list(input_stream.iter_data()) + except BaseException: + # A run that raised defines no rows, as on the async path (ITL-1121). + self.clear_cache() + raise self._hold(rows, load_only=True) return rows try: diff --git a/tests/test_pipeline/test_node_view_review_fixes.py b/tests/test_pipeline/test_node_view_review_fixes.py index 1e1470592..b181ae897 100644 --- a/tests/test_pipeline/test_node_view_review_fixes.py +++ b/tests/test_pipeline/test_node_view_review_fixes.py @@ -224,3 +224,80 @@ def test_a_job_with_no_function_node_says_what_ran(how: str) -> None: if how != "node.run": # a held node.run() makes no summary (see above) second = _run_job_or_node(job, how) assert _NOTHING_TO_DO in repr(second).splitlines()[0] + + +# --------------------------------------------------------------------------- +# Comment 5: a load-only execute() whose input raises clears the side-effect node +# --------------------------------------------------------------------------- + + +@function_pod(output_keys="z") +def _boom(y: int) -> int: + """Always raise.""" + raise RuntimeError("input failed") + + +def _failing_y_stream() -> Any: + """A pod stream outside a job (tag ``k``, data ``z``) that raises when read.""" + import pyarrow as pa + + from orcapod.core.streams import ArrowTableStream + + schema = pa.schema( + [ + pa.field("k", pa.int64(), nullable=False), + pa.field("y", pa.int64(), nullable=False), + ] + ) + return _boom.pod( + ArrowTableStream( + pa.table({"k": [1], "y": [1]}, schema=schema), tag_columns=["k"] + ) + ) + + +class _FailingChannel: + """An input channel whose iteration raises.""" + + def __aiter__(self) -> _FailingChannel: + return self + + async def __anext__(self) -> Any: + raise RuntimeError("input failed") + + +@pytest.mark.parametrize("path", ["sync", "async"]) +@pytest.mark.parametrize("load_only", [False, True]) +def test_an_execute_whose_input_raises_leaves_the_side_effect_node_stale( + path: str, load_only: bool +) -> None: + """The node and the node below it read as not run, whatever the run kind. + + ``execute``/``async_execute`` are the node's run entry points (orchestrators call + them); this test is about them, so it calls them directly. + """ + import asyncio + + from orcapod.channels import Channel + from orcapod.side_effects import tap_pod + + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + rows = inc.pod(_source(), label="rowwise") + tapped: Any = tap_pod(_deliver).process(rows, label="tap") + tapped.select_data_columns(["y"], label="down") + job.run() + tap, down = job.nodes["tap"], job.nodes["down"] + # The async delivery path reads its input inside a task group, which raises the + # input's error in an ExceptionGroup. + with pytest.raises((RuntimeError, ExceptionGroup)): + if path == "sync": + tap.execute(_failing_y_stream(), load_only=load_only) + else: + out: Channel[Any] = Channel() + asyncio.run( + tap.async_execute([_FailingChannel()], out.writer, load_only=load_only) + ) + assert tap.is_stale and down.is_stale + assert tap.as_table().num_rows == 0 + assert down.as_table().num_rows == 0 From e3d4984bdbdf296b195423e557f46ba8253f03d7 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 05:45:35 -0800 Subject: [PATCH 44/63] chore(repro): ruff-clean the ITL-1121 reproduction scripts Co-Authored-By: Claude Opus 5.5 --- .../c1_observer_reads_in_on_node_end.py | 12 +++++- .../itl_1121/c2_operator_lazy_failure.py | 28 ++++++++++--- .../itl_1121/c3_node_run_pending_writes.py | 41 +++++++++++++++---- .../itl_1121/c4_summary_side_effect.py | 34 ++++++++++++--- .../itl_1121/c5_load_only_execute_failure.py | 22 ++++++++-- .../c6_load_only_over_side_effect_stream.py | 25 ++++++++--- 6 files changed, 130 insertions(+), 32 deletions(-) diff --git a/superpowers/scripts/itl_1121/c1_observer_reads_in_on_node_end.py b/superpowers/scripts/itl_1121/c1_observer_reads_in_on_node_end.py index b74c8230b..0f4406a0d 100644 --- a/superpowers/scripts/itl_1121/c1_observer_reads_in_on_node_end.py +++ b/superpowers/scripts/itl_1121/c1_observer_reads_in_on_node_end.py @@ -32,11 +32,19 @@ def on_node_end(self, node_label: str, node_hash: str) -> None: def build(db: Any) -> PipelineJob: job = PipelineJob(name="j", database=db) with job: - inc.pod(DictSource([{"k": i, "x": i} for i in range(3)], tag_columns=["k"], source_id="s"), label="rowwise") + inc.pod( + DictSource( + [{"k": i, "x": i} for i in range(3)], tag_columns=["k"], source_id="s" + ), + label="rowwise", + ) return job -for name, orch in [("sync", SyncPipelineOrchestrator), ("async", AsyncPipelineOrchestrator)]: +for name, orch in [ + ("sync", SyncPipelineOrchestrator), + ("async", AsyncPipelineOrchestrator), +]: for load_only in (False, True): db = InMemoryArrowDatabase() if load_only: diff --git a/superpowers/scripts/itl_1121/c2_operator_lazy_failure.py b/superpowers/scripts/itl_1121/c2_operator_lazy_failure.py index a93767b7d..1a4c78100 100644 --- a/superpowers/scripts/itl_1121/c2_operator_lazy_failure.py +++ b/superpowers/scripts/itl_1121/c2_operator_lazy_failure.py @@ -20,19 +20,33 @@ def flaky(x: int) -> int: return x + 1 -schema = pa.schema([pa.field("id", pa.int64(), nullable=False), pa.field("x", pa.int64(), nullable=False)]) -base = ArrowTableStream(pa.table({"id": [1, 2], "x": [1, 2]}, schema=schema), tag_columns=["id"]) +schema = pa.schema( + [ + pa.field("id", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] +) +base = ArrowTableStream( + pa.table({"id": [1, 2], "x": [1, 2]}, schema=schema), tag_columns=["id"] +) for label, db in [("no database", None), ("CacheMode.OFF", InMemoryArrowDatabase())]: FAIL["on"] = True upstream = flaky.pod(base) # a pod stream outside a job: lazy, raises when read - node = OperatorJobNode(operator=MapData({"y": "z"}), input_streams=(upstream,), pipeline_database=db, cache_mode=CacheMode.OFF) + node = OperatorJobNode( + operator=MapData({"y": "z"}), + input_streams=(upstream,), + pipeline_database=db, + cache_mode=CacheMode.OFF, + ) try: node.run() except Exception as exc: print(f"{label}: run() raised {type(exc).__name__}: {exc}") - print(f" after the failed run: holds_current_run={node.holds_current_run(load_only=False)} is_stale={node.is_stale} " - f"kept output installed={node._cached_output_stream is not None}") # private: what the comment is about + print( + f" after the failed run: holds_current_run={node.holds_current_run(load_only=False)} is_stale={node.is_stale} " + f"kept output installed={node._cached_output_stream is not None}" + ) # private: what the comment is about try: print(f" as_table() rows={node.as_table().num_rows}") except Exception as exc: @@ -40,6 +54,8 @@ def flaky(x: int) -> int: print(f" head(): {node.head().status}") FAIL["on"] = False try: - print(f" second run() after the input recovers: rows={node.run().as_table().num_rows}") + print( + f" second run() after the input recovers: rows={node.run().as_table().num_rows}" + ) except Exception as exc: print(f" second run() raised {type(exc).__name__}: {exc}") diff --git a/superpowers/scripts/itl_1121/c3_node_run_pending_writes.py b/superpowers/scripts/itl_1121/c3_node_run_pending_writes.py index c4da26214..1e359fb5a 100644 --- a/superpowers/scripts/itl_1121/c3_node_run_pending_writes.py +++ b/superpowers/scripts/itl_1121/c3_node_run_pending_writes.py @@ -6,7 +6,6 @@ committed pipeline-record version only references committed results after node.run(). """ -import sys import tempfile from pathlib import Path @@ -25,7 +24,9 @@ def double(x: int) -> int: def col(table: deltalake.DeltaTable, name: str) -> set: - return set(delta_to_pyarrow_dataset(table).to_table(columns=[name])[name].to_pylist()) + return set( + delta_to_pyarrow_dataset(table).to_table(columns=[name])[name].to_pylist() + ) def check(root: Path) -> None: @@ -36,12 +37,25 @@ def check(root: Path) -> None: } result_history = deltalake.DeltaTable(str(tables["result"])).history() pipeline_history = deltalake.DeltaTable(str(tables["pipeline"])).history() - print(f" pipeline commits: {len(pipeline_history)}, result commits: {len(result_history)}") + print( + f" pipeline commits: {len(pipeline_history)}, result commits: {len(result_history)}" + ) bad = 0 for entry in pipeline_history: - records = deltalake.DeltaTable(str(tables["pipeline"]), version=entry["version"]) - versions = [r["version"] for r in result_history if r["timestamp"] <= entry["timestamp"]] - committed = col(deltalake.DeltaTable(str(tables["result"]), version=max(versions)), "__record_id") if versions else set() + records = deltalake.DeltaTable( + str(tables["pipeline"]), version=entry["version"] + ) + versions = [ + r["version"] for r in result_history if r["timestamp"] <= entry["timestamp"] + ] + committed = ( + col( + deltalake.DeltaTable(str(tables["result"]), version=max(versions)), + "__record_id", + ) + if versions + else set() + ) if not col(records, "__data_id") <= committed: bad += 1 print(f" pipeline versions referencing uncommitted results: {bad}") @@ -52,10 +66,21 @@ def check(root: Path) -> None: root = Path(tmp) store = DeltaTableDatabase(root, batch_size=batch_size) for i in range(batch_size - 1): # pending, never flushed - store.add_record(("leftover",), f"left-{i}".encode(), pa.table({"v": pa.array([i], pa.int64())})) + store.add_record( + ("leftover",), + f"left-{i}".encode(), + pa.table({"v": pa.array([i], pa.int64())}), + ) job = PipelineJob(name="j", database=store) with job: - double.pod(DictSource([{"k": i, "x": i} for i in range(n)], tag_columns=["k"], source_id="s"), label="dbl") + double.pod( + DictSource( + [{"k": i, "x": i} for i in range(n)], + tag_columns=["k"], + source_id="s", + ), + label="dbl", + ) print(f"batch_size={batch_size}, rows={n}: node.run()") job.nodes["dbl"].run() check(root) diff --git a/superpowers/scripts/itl_1121/c4_summary_side_effect.py b/superpowers/scripts/itl_1121/c4_summary_side_effect.py index 980c76d2e..2c7e8157c 100644 --- a/superpowers/scripts/itl_1121/c4_summary_side_effect.py +++ b/superpowers/scripts/itl_1121/c4_summary_side_effect.py @@ -20,12 +20,22 @@ def deliver(y: int, ctx: Any) -> None: DELIVERED.append(y) -for name, orch in [("sync", SyncPipelineOrchestrator), ("async", AsyncPipelineOrchestrator)]: +for name, orch in [ + ("sync", SyncPipelineOrchestrator), + ("async", AsyncPipelineOrchestrator), +]: for via in ("job.run", "node.run"): DELIVERED.clear() job = PipelineJob(name="j", database=InMemoryArrowDatabase()) with job: - rows = inc.pod(DictSource([{"k": i, "x": i} for i in range(3)], tag_columns=["k"], source_id="s"), label="rowwise") + rows = inc.pod( + DictSource( + [{"k": i, "x": i} for i in range(3)], + tag_columns=["k"], + source_id="s", + ), + label="rowwise", + ) tap_pod(deliver).process(rows, label="tap") job.run(orchestrator=orch()) first = len(DELIVERED) @@ -38,17 +48,29 @@ def deliver(y: int, ctx: Any) -> None: continue # node.run() is sync only job.nodes["tap"].run() summary = job.nodes["tap"].last_run - print(f"--- {name} {via}: first run delivered {first}; second run delivered {len(DELIVERED)}") + print( + f"--- {name} {via}: first run delivered {first}; second run delivered {len(DELIVERED)}" + ) print(repr(summary)) # The same with an operator node below the held function node instead of a side effect. -for name, orch in [("sync", SyncPipelineOrchestrator), ("async", AsyncPipelineOrchestrator)]: +for name, orch in [ + ("sync", SyncPipelineOrchestrator), + ("async", AsyncPipelineOrchestrator), +]: job = PipelineJob(name="j", database=InMemoryArrowDatabase()) with job: - rows = inc.pod(DictSource([{"k": i, "x": i} for i in range(3)], tag_columns=["k"], source_id="s"), label="rowwise") + rows = inc.pod( + DictSource( + [{"k": i, "x": i} for i in range(3)], tag_columns=["k"], source_id="s" + ), + label="rowwise", + ) rows.select_data_columns(["y"], label="op") job.run(orchestrator=orch()) job.nodes["op"].clear_cache() summary = job.run(orchestrator=orch()) - print(f"--- {name} job.run, operator node cleared: op holds_current_run={job.nodes['op'].holds_current_run(load_only=False)}") + print( + f"--- {name} job.run, operator node cleared: op holds_current_run={job.nodes['op'].holds_current_run(load_only=False)}" + ) print(repr(summary)) diff --git a/superpowers/scripts/itl_1121/c5_load_only_execute_failure.py b/superpowers/scripts/itl_1121/c5_load_only_execute_failure.py index 55ea3f5d7..76c0c9ff8 100644 --- a/superpowers/scripts/itl_1121/c5_load_only_execute_failure.py +++ b/superpowers/scripts/itl_1121/c5_load_only_execute_failure.py @@ -31,14 +31,26 @@ def deliver(y: int, ctx: Any) -> None: def build() -> PipelineJob: job = PipelineJob(name="j", database=InMemoryArrowDatabase()) with job: - rows = inc.pod(DictSource([{"k": i, "x": i} for i in range(3)], tag_columns=["k"], source_id="s"), label="rowwise") + rows = inc.pod( + DictSource( + [{"k": i, "x": i} for i in range(3)], tag_columns=["k"], source_id="s" + ), + label="rowwise", + ) tapped: Any = tap_pod(deliver).process(rows, label="tap") tapped.select_data_columns(["y"], label="down") return job -schema = pa.schema([pa.field("k", pa.int64(), nullable=False), pa.field("y", pa.int64(), nullable=False)]) -failing = boom.pod(ArrowTableStream(pa.table({"k": [1], "y": [1]}, schema=schema), tag_columns=["k"])) +schema = pa.schema( + [ + pa.field("k", pa.int64(), nullable=False), + pa.field("y", pa.int64(), nullable=False), + ] +) +failing = boom.pod( + ArrowTableStream(pa.table({"k": [1], "y": [1]}, schema=schema), tag_columns=["k"]) +) class FailingChannel: @@ -60,7 +72,9 @@ async def __anext__(self): tap.execute(failing, load_only=True) else: out = Channel() - asyncio.run(tap.async_execute([FailingChannel()], out.writer, load_only=True)) + asyncio.run( + tap.async_execute([FailingChannel()], out.writer, load_only=True) + ) except Exception as exc: print(f"{mode}: raised {type(exc).__name__}: {exc}") print( diff --git a/superpowers/scripts/itl_1121/c6_load_only_over_side_effect_stream.py b/superpowers/scripts/itl_1121/c6_load_only_over_side_effect_stream.py index 89ffe095a..afaddfeb6 100644 --- a/superpowers/scripts/itl_1121/c6_load_only_over_side_effect_stream.py +++ b/superpowers/scripts/itl_1121/c6_load_only_over_side_effect_stream.py @@ -39,14 +39,25 @@ def dbl(y: int) -> int: return 2 * y -schema = pa.schema([pa.field("k", pa.int64(), nullable=False), pa.field("x", pa.int64(), nullable=False)]) -base = ArrowTableStream(pa.table({"k": [1, 2, 3], "x": [1, 2, 3]}, schema=schema), tag_columns=["k"]) - -upstream: Any = tap_pod(deliver_up).process(base) # a SideEffectPodStream, outside a job +schema = pa.schema( + [ + pa.field("k", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] +) +base = ArrowTableStream( + pa.table({"k": [1, 2, 3], "x": [1, 2, 3]}, schema=schema), tag_columns=["k"] +) + +upstream: Any = tap_pod(deliver_up).process( + base +) # a SideEffectPodStream, outside a job print("input type:", type(upstream).__name__) node = SideEffectJobNode(tap_pod(deliver_own), upstream) view = node.run(load_only=True) -print(f"side-effect node, run(load_only=True): upstream delivered {len(UP)}, own delivered {len(OWN)}, rows {view.as_table().num_rows}") +print( + f"side-effect node, run(load_only=True): upstream delivered {len(UP)}, own delivered {len(OWN)}, rows {view.as_table().num_rows}" +) node.clear_cache() node.run(load_only=True) print(f" again after clear_cache(): upstream delivered {len(UP)} in total") @@ -54,4 +65,6 @@ def dbl(y: int) -> int: fn_upstream: Any = inc.pod(base) # a FunctionPodStream, outside a job fnode = FunctionJobNode(dbl.pod, fn_upstream, pipeline_database=InMemoryArrowDatabase()) fview = fnode.run(load_only=True) -print(f"function node over a pod stream, run(load_only=True): upstream function called {len(CALLS)} times; status {fview.status}") +print( + f"function node over a pod stream, run(load_only=True): upstream function called {len(CALLS)} times; status {fview.status}" +) From 16a6210553d86324135c65da4e960b774fa3d36d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 05:51:11 -0800 Subject: [PATCH 45/63] fix(node_run): a load-only run refuses an input that delivers when read (ITL-1121) A job node built over a side-effect pod's stream outside a job (SideEffectPodStream) read it in node.run(load_only=True), which delivered every row although a load-only run delivers nothing. The run plan now refuses such an input with a ValueError before any stream is read, for function, operator and side-effect job nodes, as the node run or in the cascade of a node below it. A full run is unchanged. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/function_node.py | 8 +- src/orcapod/core/nodes/operator_node.py | 3 + src/orcapod/pipeline/node_run.py | 40 +++++++- .../test_node_view_review_fixes.py | 96 +++++++++++++++++++ 4 files changed, 145 insertions(+), 2 deletions(-) diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index f3339300c..f3cb27c04 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -300,7 +300,11 @@ def run_job_node( ``RunSummary`` or ``None``. caller: The frame of the ``run()`` caller. """ - from orcapod.pipeline.node_run import depends_on_job_nodes, run_with_dependencies + from orcapod.pipeline.node_run import ( + depends_on_job_nodes, + refuse_delivering_inputs, + run_with_dependencies, + ) from orcapod.pipeline.run_summary import warn_row_failures if node.holds_current_run(load_only=load_only): @@ -308,6 +312,8 @@ def run_job_node( if depends_on_job_nodes(node): summary = run_with_dependencies(node, load_only=load_only) else: + if load_only: + refuse_delivering_inputs(node) summary = run_alone() if summary is None: return diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index e32c592a8..21e5e7595 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -1531,6 +1531,7 @@ def run(self, *, load_only: bool = False) -> StreamView: """ from orcapod.pipeline.node_run import ( depends_on_job_nodes, + refuse_delivering_inputs, run_with_dependencies, ) from orcapod.pipeline.run_summary import warn_row_failures @@ -1547,6 +1548,8 @@ def run(self, *, load_only: bool = False) -> StreamView: return self._run_view(computed=False) if self._loaded: raise self._cannot_compute_error() + if load_only: + refuse_delivering_inputs(self) if self.is_stale or not self._holds_run(load_only=load_only): # Stale, or a load-only run's output asked for by a full run: run again. self.clear_cache() diff --git a/src/orcapod/pipeline/node_run.py b/src/orcapod/pipeline/node_run.py index bdeb0b507..1369f55f8 100644 --- a/src/orcapod/pipeline/node_run.py +++ b/src/orcapod/pipeline/node_run.py @@ -48,6 +48,33 @@ def depends_on_job_nodes(node: Any) -> bool: return any(_is_job_node(upstream) for upstream in node.upstreams) +def refuse_delivering_inputs(node: Any) -> None: + """Refuse a load-only run of ``node`` over an input that delivers when it is read. + + A load-only run computes, delivers and writes nothing (ITL-1095). The stream a + side-effect pod returns outside a job (``SideEffectPodStream``) delivers its side + effects on every read, so a load-only run cannot read it; it is refused before + anything is read (ITL-1121). Another stream outside a job is read as it always is. + + Args: + node: A job node a load-only run would run. + + Raises: + ValueError: If an input of ``node`` is a ``SideEffectPodStream``. + """ + from orcapod.side_effects import SideEffectPodStream + + for upstream in node.upstreams: + if isinstance(upstream, SideEffectPodStream): + raise ValueError( + f"A load-only run cannot run node {node.label!r}: its input " + f"{upstream.label!r} is a side-effect pod's stream outside a job, which " + "delivers its side effects when it is read, and a load-only run " + "delivers nothing. Run it without load_only, or build the side-effect " + "pod inside the job." + ) + + def _stores(nodes: list[Any]) -> list[ArrowDatabaseProtocol]: """The distinct stores the given nodes write to: pipeline and result databases.""" found: list[ArrowDatabaseProtocol] = [] @@ -114,7 +141,8 @@ def plan_run(graph: Any, *, load_only: bool, outside: Sequence[Any] = ()) -> Run is read only when a node that runs reads it; a stream outside the job (``outside``) is read for the nodes it feeds; every other node runs. The same plan serves ``job.run()`` (its whole graph) and ``node.run()`` (the node and what it - depends on). + depends on). A load-only plan refuses, before reading anything, a node to run whose + input delivers when read (``refuse_delivering_inputs``). Args: graph: The run's node graph (``topological_sort``, ``predecessors``). @@ -124,6 +152,9 @@ def plan_run(graph: Any, *, load_only: bool, outside: Sequence[Any] = ()) -> Run Returns: The plan. + + Raises: + ValueError: As ``refuse_delivering_inputs``, for a load-only run. """ from orcapod.core.nodes.source_node import SourceJobNode from orcapod.core.nodes.staleness import RunHoldingNode @@ -154,6 +185,11 @@ def is_source(node: Any) -> bool: and not is_outside(node) and not any(node is c for c in current) ] + if load_only: + # Refused before any stream is read below: a load-only run cannot read an + # input that delivers (ITL-1121). + for node in to_run: + refuse_delivering_inputs(node) included: list[Any] = [] for node in order: feeds_a_run = any( @@ -222,6 +258,8 @@ def run_with_dependencies(target: Any, *, load_only: bool) -> RunSummary: Raises: StoreWriteError: If committing the run's writes failed. + ValueError: For a load-only run, if a node it would run reads an input that + delivers when read (``refuse_delivering_inputs``); nothing has run then. """ from orcapod.core.nodes.source_node import SourceJobNode from orcapod.pipeline.dag import OrcaDAG diff --git a/tests/test_pipeline/test_node_view_review_fixes.py b/tests/test_pipeline/test_node_view_review_fixes.py index b181ae897..adf5d7dd8 100644 --- a/tests/test_pipeline/test_node_view_review_fixes.py +++ b/tests/test_pipeline/test_node_view_review_fixes.py @@ -301,3 +301,99 @@ def test_an_execute_whose_input_raises_leaves_the_side_effect_node_stale( assert tap.is_stale and down.is_stale assert tap.as_table().num_rows == 0 assert down.as_table().num_rows == 0 + + +# --------------------------------------------------------------------------- +# Comment 6: a load-only run refuses an input that delivers when it is read +# --------------------------------------------------------------------------- + +_UPSTREAM_DELIVERED: list[int] = [] + + +def _deliver_upstream(x: int, ctx: Any) -> None: + """The side effect of the stream outside a job: record the delivered value.""" + _UPSTREAM_DELIVERED.append(x) + + +def _deliver_own(x: int, ctx: Any) -> None: + """The side effect of the node under test: record the delivered value.""" + _DELIVERED.append(x) + + +@function_pod(output_keys="w") +def _twice(x: int) -> int: + """Return ``2 * x``.""" + return 2 * x + + +def _delivering_stream() -> Any: + """A side-effect pod's stream outside a job: reading it delivers each row.""" + from orcapod.side_effects import tap_pod + + return tap_pod(_deliver_upstream).process(_base_stream()) + + +def _node_over(kind: str, stream: Any) -> Any: + """A job node of ``kind`` built directly over ``stream`` (no job).""" + from orcapod.core.nodes.function_node import FunctionJobNode + from orcapod.core.nodes.operator_node import OperatorJobNode + from orcapod.core.operators import MapData + from orcapod.side_effects import SideEffectJobNode, tap_pod + from orcapod.types import CacheMode + + if kind == "side_effect": + return SideEffectJobNode(tap_pod(_deliver_own), stream) + if kind == "function": + return FunctionJobNode( + _twice.pod, stream, pipeline_database=InMemoryArrowDatabase() + ) + return OperatorJobNode( + operator=MapData({"x": "renamed"}), + input_streams=(stream,), + pipeline_database=InMemoryArrowDatabase(), + cache_mode=CacheMode.OFF, + ) + + +@pytest.mark.parametrize("kind", ["side_effect", "function", "operator"]) +def test_a_load_only_run_refuses_an_input_that_delivers(kind: str) -> None: + """Refused before the input is read: nothing is delivered and the node holds no run.""" + _UPSTREAM_DELIVERED.clear() + _DELIVERED.clear() + node = _node_over(kind, _delivering_stream()) + with pytest.raises(ValueError, match="delivers its side effects when it is read"): + node.run(load_only=True) + assert _UPSTREAM_DELIVERED == [] + assert _DELIVERED == [] + assert not node.holds_current_run(load_only=True) + + +@pytest.mark.parametrize("kind", ["side_effect", "function", "operator"]) +def test_a_load_only_cascade_refuses_an_input_that_delivers(kind: str) -> None: + """The input that delivers feeds a node the cascade would run first.""" + from orcapod.core.nodes.operator_node import OperatorJobNode + from orcapod.core.operators import DropNulls + from orcapod.types import CacheMode + + _UPSTREAM_DELIVERED.clear() + first = _node_over(kind, _delivering_stream()) + target = OperatorJobNode( + operator=DropNulls(), + input_streams=(first,), + pipeline_database=InMemoryArrowDatabase(), + cache_mode=CacheMode.OFF, + ) + with pytest.raises(ValueError, match="delivers its side effects when it is read"): + target.run(load_only=True) + assert _UPSTREAM_DELIVERED == [] + assert not first.holds_current_run(load_only=True) + assert not target.holds_current_run(load_only=True) + + +@pytest.mark.parametrize("kind", ["side_effect", "function", "operator"]) +def test_a_full_run_over_an_input_that_delivers_reads_it(kind: str) -> None: + """Unchanged: a full run reads the stream outside a job, which delivers.""" + _UPSTREAM_DELIVERED.clear() + node = _node_over(kind, _delivering_stream()) + assert node.run().as_table().num_rows == 2 + assert len(_UPSTREAM_DELIVERED) == 2 From 2c0f1f845b0b8aca2a4bfe2f83a8a642bb014e3a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 05:51:37 -0800 Subject: [PATCH 46/63] docs(function_pods): a held side-effect node delivers nothing on a second run (ITL-1121) Also assert cache reuse with a recording function in test_caching_flows, and record progress in the plan. Co-Authored-By: Claude Opus 5.5 --- docs/concepts/function-pods.md | 8 +++++--- .../2026-11-14-itl-1121-node-view-review-fixes.md | 11 +++++++++++ test-objective/integration/test_caching_flows.py | 6 ++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 556ef5b56..a0c53235b 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -832,14 +832,16 @@ notify_job.run() node = notify_job.nodes["notify"] print(node.as_table().num_rows, len(list(node.iter_data())), len(sent)) # 3 3 3 -node.run() # delivers only what the invocation log does not record: nothing here +node.run() # holds a current run: runs nothing and delivers nothing print(len(sent)) # 3 ``` - With `track_completion=True` (the default), a run delivers only the rows whose delivery the - node's invocation log does not record. With `track_completion=False`, every run, a second - `job.run()` or `node.run()`, delivers every row again; reads still deliver nothing. + node's invocation log does not record. With `track_completion=False`, a run delivers every + row. A node that holds a current run is not run again, so a second `job.run()` or + `node.run()` delivers nothing; call `clear_cache()` on the node first to deliver again. + Reads never deliver. - The rows a read returns are those the last run in this process passed on: under `on_error="log"`, a row whose delivery failed is left out with `drop_on_failure=True` (the default) and kept with `drop_on_failure=False`, as for the nodes downstream. diff --git a/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md b/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md index 7adffc20c..0d364d2e1 100644 --- a/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md +++ b/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md @@ -137,3 +137,14 @@ Reproduction scripts: `superpowers/scripts/itl_1121/c1..c6_*.py` (committed). ## Progress notes - 2026-11-14: reproduction table sent and approved (see decisions). Scripts committed (86902ec0). +- 2026-11-14 (session stopped at usage limit): DONE and pushed: fixes 1, 2, 4, 5, 6 (each its own + commit, tests in tests/test_pipeline/test_node_view_review_fixes.py, 32 pass), docs 8, test 9. + Fix 6 lives in node_run.plan_run (refuse_delivering_inputs) because plan_run reads outside + streams as held rows; plus run_job_node / OperatorJobNode.run alone paths. + Observation for the coordinator: a node.run() of a node holding a current run makes no summary; + node.last_run stays the previous run's (existing #380 behaviour, not changed). + LEFT: changelog.d/ITL-1121.md (### Fixed), design_issues/ITL-1121-NV1.md, the FunctionPodStream + three lines for the coordinator (repro in superpowers/scripts/itl_1121/c6_*.py), basedpyright, + full `uv run pytest tests/` + test-objective, draft PR (top line "Stacked on #380: merge after + it. Fixes ITL-1121"), orca linear attach / worktree set --pr, signed replies on the nine #381 + findings (3: evidence c3; 7: ITL-1097), CI green, CodeRabbit round, worker_done. diff --git a/test-objective/integration/test_caching_flows.py b/test-objective/integration/test_caching_flows.py index ed91d25e4..34d6a051b 100644 --- a/test-objective/integration/test_caching_flows.py +++ b/test-objective/integration/test_caching_flows.py @@ -23,7 +23,11 @@ # --------------------------------------------------------------------------- +_DOUBLE_CALLS: list[int] = [] + + def _double(x: int) -> int: + _DOUBLE_CALLS.append(x) return x * 2 @@ -86,9 +90,11 @@ def test_second_run_uses_cache(self): result_database=result_db, ) # A node shows rows only after a run (ITL-1095); this run reads them all. + _DOUBLE_CALLS.clear() node2.run() data = list(node2.iter_data()) assert len(data) == 3 + assert _DOUBLE_CALLS == [] # every result came from the store class TestDerivedSourceReingestion: From 902e4e9bd36941efbc06c85a600faf7441d9f76a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 06:31:15 -0800 Subject: [PATCH 47/63] docs(changelog): ITL-1121 fragment and design issue entry Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1121.md | 7 +++++++ design_issues/ITL-1121-NV1.md | 31 +++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 changelog.d/ITL-1121.md create mode 100644 design_issues/ITL-1121-NV1.md diff --git a/changelog.d/ITL-1121.md b/changelog.d/ITL-1121.md new file mode 100644 index 000000000..245d809f0 --- /dev/null +++ b/changelog.d/ITL-1121.md @@ -0,0 +1,7 @@ +### Fixed + +- An observer that read a function job node in `on_node_end` emptied it: the node was marked current only after that hook, so the read found it stale and cleared the run's rows (ITL-1121). The node is now current before the hook, on sync, async and load-only runs. +- An operator job node without a database, or in `CacheMode.OFF`, whose output failed to compute in `run()` kept claiming a current run and its reads raised; it now reads as not run, like every node whose run raised. +- The run summary said "nothing to do; every node holds a current run" when the run ran a side-effect node (and delivered) or an operator node below a held function node. It now says so only when the run ran no node; `RunSummary` takes `ran_nothing`. +- A side-effect job node whose load-only `execute()` failed on its input stayed current, and so did the nodes below it; it is now cleared, as on the async path. +- `node.run(load_only=True)` of a job node built over a side-effect pod's stream outside a job read that stream, which delivered every row. A load-only run now refuses such an input with a `ValueError` before anything is read; a run without `load_only` is unchanged. diff --git a/design_issues/ITL-1121-NV1.md b/design_issues/ITL-1121-NV1.md new file mode 100644 index 000000000..396bcf9f6 --- /dev/null +++ b/design_issues/ITL-1121-NV1.md @@ -0,0 +1,31 @@ +## `src/orcapod/core/nodes/function_node.py` + +### NV1 — Job node view: a run's rows lost to an observer, failed runs left current, a misleading summary +**Status:** resolved +**Severity:** medium +**Issue:** ITL-1121 + +A full review of the job node current-run view (ITL-1095) found five defects, each reproduced +through the public interface (`superpowers/scripts/itl_1121/`): + +- A function job node was stamped as modified after `on_node_end`, so an observer reading the + node there found it stale and cleared the run's rows; the node read empty after the run (sync, + async and both load-only paths). +- An operator job node without a database, or in `CacheMode.OFF`, computed its lazy output while + building the `run()` view, outside the block that clears a failed run: after a failure it + claimed a current run and its reads raised. +- `RunSummary` said "nothing to do; every node holds a current run" when the run ran only a + side-effect node (which delivered) or an operator node below a held function node: the + headline judged only the function nodes' counts. +- A side-effect job node's sync `execute(load_only=True)` whose input raised kept its modified + time, so it and the nodes below it were not stale. +- `node.run(load_only=True)` over a side-effect pod's stream outside a job read it, which + delivered every row, although a load-only run delivers nothing. + +**Fix:** stamp before `on_node_end`; build the computed view inside the clearing block; +`RunSummary(ran_nothing=...)` from the run plan; clear the node when the load-only input +raises; the run plan refuses (`ValueError`, before reading) a load-only run whose node reads a +`SideEffectPodStream` (`node_run.refuse_delivering_inputs`). Two review comments did not +reproduce or are tracked elsewhere: a node run outside `job.run()` commits every row, so no +pre-run flush is needed; a function node with no database holding no rows after an async run is +ITL-1097. From de87d1626c7fa0bf4d7c24b67f3fd37e779d965f Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 06:33:20 -0800 Subject: [PATCH 48/63] test(itl-1121): type-clean the regression tests and reproduction scripts Co-Authored-By: Claude Opus 5.5 --- .../itl_1121/c3_node_run_pending_writes.py | 2 +- .../itl_1121/c4_summary_side_effect.py | 6 ++-- .../itl_1121/c5_load_only_execute_failure.py | 3 +- .../c6_load_only_over_side_effect_stream.py | 30 +++++++++-------- .../test_node_view_review_fixes.py | 32 +++++++++++-------- 5 files changed, 43 insertions(+), 30 deletions(-) diff --git a/superpowers/scripts/itl_1121/c3_node_run_pending_writes.py b/superpowers/scripts/itl_1121/c3_node_run_pending_writes.py index 1e359fb5a..2cce0f305 100644 --- a/superpowers/scripts/itl_1121/c3_node_run_pending_writes.py +++ b/superpowers/scripts/itl_1121/c3_node_run_pending_writes.py @@ -23,7 +23,7 @@ def double(x: int) -> int: return 2 * x -def col(table: deltalake.DeltaTable, name: str) -> set: +def col(table: deltalake.DeltaTable, name: str) -> set[object]: return set( delta_to_pyarrow_dataset(table).to_table(columns=[name])[name].to_pylist() ) diff --git a/superpowers/scripts/itl_1121/c4_summary_side_effect.py b/superpowers/scripts/itl_1121/c4_summary_side_effect.py index 2c7e8157c..8c40fe8f4 100644 --- a/superpowers/scripts/itl_1121/c4_summary_side_effect.py +++ b/superpowers/scripts/itl_1121/c4_summary_side_effect.py @@ -36,7 +36,8 @@ def deliver(y: int, ctx: Any) -> None: ), label="rowwise", ) - tap_pod(deliver).process(rows, label="tap") + tap: Any = tap_pod(deliver) + tap.process(rows, label="tap") job.run(orchestrator=orch()) first = len(DELIVERED) job.nodes["tap"].clear_cache() @@ -66,7 +67,8 @@ def deliver(y: int, ctx: Any) -> None: ), label="rowwise", ) - rows.select_data_columns(["y"], label="op") + rows_any: Any = rows + rows_any.select_data_columns(["y"], label="op") job.run(orchestrator=orch()) job.nodes["op"].clear_cache() summary = job.run(orchestrator=orch()) diff --git a/superpowers/scripts/itl_1121/c5_load_only_execute_failure.py b/superpowers/scripts/itl_1121/c5_load_only_execute_failure.py index 76c0c9ff8..f9924d46f 100644 --- a/superpowers/scripts/itl_1121/c5_load_only_execute_failure.py +++ b/superpowers/scripts/itl_1121/c5_load_only_execute_failure.py @@ -37,7 +37,8 @@ def build() -> PipelineJob: ), label="rowwise", ) - tapped: Any = tap_pod(deliver).process(rows, label="tap") + tap: Any = tap_pod(deliver) + tapped: Any = tap.process(rows, label="tap") tapped.select_data_columns(["y"], label="down") return job diff --git a/superpowers/scripts/itl_1121/c6_load_only_over_side_effect_stream.py b/superpowers/scripts/itl_1121/c6_load_only_over_side_effect_stream.py index afaddfeb6..c2390a5e1 100644 --- a/superpowers/scripts/itl_1121/c6_load_only_over_side_effect_stream.py +++ b/superpowers/scripts/itl_1121/c6_load_only_over_side_effect_stream.py @@ -49,21 +49,25 @@ def dbl(y: int) -> int: pa.table({"k": [1, 2, 3], "x": [1, 2, 3]}, schema=schema), tag_columns=["k"] ) -upstream: Any = tap_pod(deliver_up).process( - base -) # a SideEffectPodStream, outside a job +tap_up: Any = tap_pod(deliver_up) +upstream: Any = tap_up.process(base) # a SideEffectPodStream, outside a job print("input type:", type(upstream).__name__) -node = SideEffectJobNode(tap_pod(deliver_own), upstream) -view = node.run(load_only=True) -print( - f"side-effect node, run(load_only=True): upstream delivered {len(UP)}, own delivered {len(OWN)}, rows {view.as_table().num_rows}" -) -node.clear_cache() -node.run(load_only=True) -print(f" again after clear_cache(): upstream delivered {len(UP)} in total") - +tap_own: Any = tap_pod(deliver_own) +node = SideEffectJobNode(tap_own, upstream) +# Before ITL-1121 this delivered every row of the upstream stream (3, then 6 after +# clear_cache()); since then the load-only run refuses the input before reading it. +try: + view = node.run(load_only=True) + print(f"side-effect node, run(load_only=True): rows {view.as_table().num_rows}") +except ValueError as exc: + print(f"side-effect node, run(load_only=True) refused: {exc}") +print(f" upstream delivered {len(UP)}, own delivered {len(OWN)}") + +# Unchanged (reported to the coordinator): an outside FunctionPodStream is read, so a +# load-only run computes its function. fn_upstream: Any = inc.pod(base) # a FunctionPodStream, outside a job -fnode = FunctionJobNode(dbl.pod, fn_upstream, pipeline_database=InMemoryArrowDatabase()) +dbl_pod: Any = dbl.pod +fnode = FunctionJobNode(dbl_pod, fn_upstream, pipeline_database=InMemoryArrowDatabase()) fview = fnode.run(load_only=True) print( f"function node over a pod stream, run(load_only=True): upstream function called {len(CALLS)} times; status {fview.status}" diff --git a/tests/test_pipeline/test_node_view_review_fixes.py b/tests/test_pipeline/test_node_view_review_fixes.py index adf5d7dd8..10be1920d 100644 --- a/tests/test_pipeline/test_node_view_review_fixes.py +++ b/tests/test_pipeline/test_node_view_review_fixes.py @@ -16,6 +16,7 @@ from orcapod.databases import InMemoryArrowDatabase from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator from orcapod.pipeline.observer import NoOpObserver +from orcapod.side_effects import SideEffectPod, tap_pod from orcapod.sources import DictSource ORCHESTRATORS = {"sync": SyncPipelineOrchestrator, "async": AsyncPipelineOrchestrator} @@ -27,6 +28,13 @@ def inc(x: int) -> int: return x + 1 +def _tap(fn: Any) -> SideEffectPod: + """A tap side-effect pod over ``fn``.""" + pod = tap_pod(fn) + assert isinstance(pod, SideEffectPod) + return pod + + def _source() -> DictSource: """Three rows, tag ``k``, data ``x``.""" return DictSource( @@ -160,13 +168,11 @@ def _deliver(y: int, ctx: Any) -> None: def _job_below_rowwise(below: str) -> PipelineJob: """``rowwise`` (``inc``) with a side-effect node or an operator node ``below`` it.""" - from orcapod.side_effects import tap_pod - job = PipelineJob(name="j", database=InMemoryArrowDatabase()) with job: - rows = inc.pod(_source(), label="rowwise") + rows: Any = inc.pod(_source(), label="rowwise") if below == "side_effect": - tap_pod(_deliver).process(rows, label="below") + _tap(_deliver).process(rows, label="below") else: rows.select_data_columns(["y"], label="below") return job @@ -279,12 +285,10 @@ def test_an_execute_whose_input_raises_leaves_the_side_effect_node_stale( import asyncio from orcapod.channels import Channel - from orcapod.side_effects import tap_pod - job = PipelineJob(name="j", database=InMemoryArrowDatabase()) with job: rows = inc.pod(_source(), label="rowwise") - tapped: Any = tap_pod(_deliver).process(rows, label="tap") + tapped: Any = _tap(_deliver).process(rows, label="tap") tapped.select_data_columns(["y"], label="down") job.run() tap, down = job.nodes["tap"], job.nodes["down"] @@ -326,11 +330,13 @@ def _twice(x: int) -> int: return 2 * x +# The decorated function's pod, typed for a node's constructor. +_twice_pod: Any = _twice.pod + + def _delivering_stream() -> Any: """A side-effect pod's stream outside a job: reading it delivers each row.""" - from orcapod.side_effects import tap_pod - - return tap_pod(_deliver_upstream).process(_base_stream()) + return _tap(_deliver_upstream).process(_base_stream()) def _node_over(kind: str, stream: Any) -> Any: @@ -338,14 +344,14 @@ def _node_over(kind: str, stream: Any) -> Any: from orcapod.core.nodes.function_node import FunctionJobNode from orcapod.core.nodes.operator_node import OperatorJobNode from orcapod.core.operators import MapData - from orcapod.side_effects import SideEffectJobNode, tap_pod + from orcapod.side_effects import SideEffectJobNode from orcapod.types import CacheMode if kind == "side_effect": - return SideEffectJobNode(tap_pod(_deliver_own), stream) + return SideEffectJobNode(_tap(_deliver_own), stream) if kind == "function": return FunctionJobNode( - _twice.pod, stream, pipeline_database=InMemoryArrowDatabase() + _twice_pod, stream, pipeline_database=InMemoryArrowDatabase() ) return OperatorJobNode( operator=MapData({"x": "renamed"}), From 979396f806543922ba119ab8be76ace87c5ac880 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 06:33:31 -0800 Subject: [PATCH 49/63] style: ruff format Co-Authored-By: Claude Opus 5.5 --- tests/test_pipeline/test_node_view_review_fixes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_pipeline/test_node_view_review_fixes.py b/tests/test_pipeline/test_node_view_review_fixes.py index 10be1920d..8135f9682 100644 --- a/tests/test_pipeline/test_node_view_review_fixes.py +++ b/tests/test_pipeline/test_node_view_review_fixes.py @@ -285,6 +285,7 @@ def test_an_execute_whose_input_raises_leaves_the_side_effect_node_stale( import asyncio from orcapod.channels import Channel + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) with job: rows = inc.pod(_source(), label="rowwise") From 1d66a2743b782166d8694dbaf9a35483f8470105 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 07:36:16 -0800 Subject: [PATCH 50/63] chore(repro): reproduce the PR #388 review's load-only delivery paths (ITL-1121) Co-Authored-By: Claude Opus 5.5 --- .../c8_cr388_load_only_delivery_paths.py | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 superpowers/scripts/itl_1121/c8_cr388_load_only_delivery_paths.py diff --git a/superpowers/scripts/itl_1121/c8_cr388_load_only_delivery_paths.py b/superpowers/scripts/itl_1121/c8_cr388_load_only_delivery_paths.py new file mode 100644 index 000000000..47970af74 --- /dev/null +++ b/superpowers/scripts/itl_1121/c8_cr388_load_only_delivery_paths.py @@ -0,0 +1,96 @@ +"""PR #388 CodeRabbit round 1: more ways a load-only run could read a delivering stream. + +D: a job's source slot bound to a SideEffectPodStream; job.run(load_only=True). +H: a stream outside a job that wraps a SideEffectPodStream (an operator over it). +C: a standalone OperatorJobNode without a database, run(load_only=True): its status. +""" + +from typing import Any + +import pyarrow as pa + +from orcapod import PipelineJob, function_pod +from orcapod.core.nodes.operator_node import OperatorJobNode +from orcapod.core.operators import MapData +from orcapod.core.streams import ArrowTableStream +from orcapod.databases import InMemoryArrowDatabase +from orcapod.side_effects import tap_pod +from orcapod.sources import DictSource +from orcapod.types import CacheMode + +UP: list[int] = [] + + +def deliver_up(x: int, ctx: Any) -> None: + UP.append(x) + + +@function_pod(output_keys="w") +def twice(x: int) -> int: + return 2 * x + + +schema = pa.schema( + [ + pa.field("k", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] +) + + +def base() -> Any: + return ArrowTableStream( + pa.table({"k": [1, 2], "x": [1, 2]}, schema=schema), tag_columns=["k"] + ) + + +def delivering() -> Any: + tap: Any = tap_pod(deliver_up) + return tap.process(base()) + + +# D +db = InMemoryArrowDatabase() +job = PipelineJob(name="j", database=db) +with job: + src = DictSource( + [{"k": 1, "x": 1}, {"k": 2, "x": 2}], tag_columns=["k"], source_id="s" + ) + twice.pod(src, label="tw") +job.run() +UP.clear() +try: + job.bind(sources={"s": delivering()}) + job.run(load_only=True) + print( + f"D: bound source slot, job.run(load_only=True): upstream delivered {len(UP)}" + ) +except Exception as exc: + print(f"D: {type(exc).__name__}: {exc}") + +# H +UP.clear() +wrapped: Any = MapData({"x": "x2"})(delivering()) # an operator stream outside a job +print("H: wrapped input type:", type(wrapped).__name__) +node = OperatorJobNode( + operator=MapData({"x2": "x3"}), + input_streams=(wrapped,), + pipeline_database=InMemoryArrowDatabase(), + cache_mode=CacheMode.OFF, +) +try: + node.run(load_only=True) + print( + f"H: operator node over the wrapped stream, run(load_only=True): upstream delivered {len(UP)}" + ) +except Exception as exc: + print(f"H: {type(exc).__name__}: {exc}") + +# C +op = OperatorJobNode( + operator=MapData({"x": "y"}), input_streams=(base(),), pipeline_database=None +) +view = op.run(load_only=True) +print( + f"C: no-database operator, run(load_only=True): status {view.status}; full run holds it: {op.holds_current_run(load_only=False)}" +) From 75324abc5cae320334672597941d5a39b592fc3e Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 09:24:53 -0800 Subject: [PATCH 51/63] fix(pipeline): PR #388 review round 1: more load-only delivery paths, operator and log-commit state (ITL-1121) - A load-only run refuses a side-effect pod's stream outside a job also when it is read through a bound source slot (job.run, run_async, node.run, the slot's own run) or through another stream outside a job (node_run walks the streams a run reads). - An operator job node without a database records a load-only run as one. - A standalone side-effect job node interrupted while committing its invocation log is cleared, so the next run runs and commits again. - The c3 reproduction script orders commits by the write calls, not timestamps. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1121.md | 4 +- design_issues/ITL-1121-NV1.md | 14 ++- src/orcapod/core/nodes/operator_node.py | 2 + src/orcapod/core/nodes/source_node.py | 11 ++- src/orcapod/pipeline/node_run.py | 44 ++++++++-- src/orcapod/side_effects.py | 5 ++ .../itl_1121/c3_node_run_pending_writes.py | 58 ++++++++----- .../itl_1121/c9_interrupted_log_commit.py | 57 ++++++++++++ .../test_node_view_review_fixes.py | 87 +++++++++++++++++++ 9 files changed, 248 insertions(+), 34 deletions(-) create mode 100644 superpowers/scripts/itl_1121/c9_interrupted_log_commit.py diff --git a/changelog.d/ITL-1121.md b/changelog.d/ITL-1121.md index 245d809f0..128ac6113 100644 --- a/changelog.d/ITL-1121.md +++ b/changelog.d/ITL-1121.md @@ -4,4 +4,6 @@ - An operator job node without a database, or in `CacheMode.OFF`, whose output failed to compute in `run()` kept claiming a current run and its reads raised; it now reads as not run, like every node whose run raised. - The run summary said "nothing to do; every node holds a current run" when the run ran a side-effect node (and delivered) or an operator node below a held function node. It now says so only when the run ran no node; `RunSummary` takes `ran_nothing`. - A side-effect job node whose load-only `execute()` failed on its input stayed current, and so did the nodes below it; it is now cleared, as on the async path. -- `node.run(load_only=True)` of a job node built over a side-effect pod's stream outside a job read that stream, which delivered every row. A load-only run now refuses such an input with a `ValueError` before anything is read; a run without `load_only` is unchanged. +- A load-only run delivered side effects when it read a side-effect pod's stream outside a job: as the input of a job node (`node.run(load_only=True)`), through another stream outside a job that reads it, or bound to a job's source slot (`job.run(load_only=True)`, `run_async`, a node's or the source slot's own `run(load_only=True)`). A load-only run now refuses such an input with a `ValueError` before anything is read; a run without `load_only` is unchanged. +- An operator job node without a database, run with `load_only=True`, showed its rows as a full run's (`rows` instead of `loaded_only`) and a later full `run()` took them as its own. +- A side-effect job node over a stream outside a job, with a store attached, kept its rows as a current run when its `run()` was interrupted (`KeyboardInterrupt`) while committing the invocation log, so the next `run()` neither ran nor committed again; it is now cleared. diff --git a/design_issues/ITL-1121-NV1.md b/design_issues/ITL-1121-NV1.md index 396bcf9f6..3f2072ae8 100644 --- a/design_issues/ITL-1121-NV1.md +++ b/design_issues/ITL-1121-NV1.md @@ -19,13 +19,19 @@ through the public interface (`superpowers/scripts/itl_1121/`): headline judged only the function nodes' counts. - A side-effect job node's sync `execute(load_only=True)` whose input raised kept its modified time, so it and the nodes below it were not stale. -- `node.run(load_only=True)` over a side-effect pod's stream outside a job read it, which - delivered every row, although a load-only run delivers nothing. +- A load-only run read a side-effect pod's stream outside a job (directly, through another + stream outside a job, or bound to a source slot), which delivered every row, although a + load-only run delivers nothing. +- An operator job node without a database did not record a load-only run as one. +- A standalone side-effect job node interrupted while committing its invocation log kept a + current run, so the next run neither ran nor committed. **Fix:** stamp before `on_node_end`; build the computed view inside the clearing block; `RunSummary(ran_nothing=...)` from the run plan; clear the node when the load-only input -raises; the run plan refuses (`ValueError`, before reading) a load-only run whose node reads a -`SideEffectPodStream` (`node_run.refuse_delivering_inputs`). Two review comments did not +raises; the run plan and a source slot's `run()` refuse (`ValueError`, before reading) a +load-only run that would read a `SideEffectPodStream` (`node_run.refuse_delivering_inputs`, +walking streams outside a job and bound source slots); the no-database operator branch +records `load_only`; the side-effect node's log commit clears the node on any interruption. Two review comments did not reproduce or are tracked elsewhere: a node run outside `job.run()` commits every row, so no pre-run flush is needed; a function node with no database holding no rows after an async run is ITL-1097. diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 21e5e7595..e4f08be20 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -1574,6 +1574,8 @@ def run(self, *, load_only: bool = False) -> StreamView: self._cached_output_stream = self._operator.process( *self._input_streams, ) + # As in the branches with a database (ITL-1121). + self._run_load_only = load_only self._update_modified_time() computed = True if computed: diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 96353607b..f4ac4d8d8 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -594,15 +594,22 @@ def __init__( def run(self, *, load_only: bool = False) -> StreamView: """Read the bound source's rows and return a ``StreamView`` of all of them. - A source is read, never computed, so ``load_only`` changes nothing; a job - node's ``run()`` takes it (ITL-1095). + A source is read, never computed, so ``load_only`` changes what is read only + for a bound stream that delivers side effects when read, which a load-only run + refuses (ITL-1121); a job node's ``run()`` takes it (ITL-1095). Args: load_only: Accepted for every job node; a source reads its rows either way. Raises: UnboundSourceError: When no concrete source is bound. + ValueError: If ``load_only`` and the bound stream delivers side effects + when read (``node_run.refuse_delivering_inputs``). """ + if load_only: + from orcapod.pipeline.node_run import refuse_delivering_inputs + + refuse_delivering_inputs(self) return super().run() @property diff --git a/src/orcapod/pipeline/node_run.py b/src/orcapod/pipeline/node_run.py index 1369f55f8..48c5125de 100644 --- a/src/orcapod/pipeline/node_run.py +++ b/src/orcapod/pipeline/node_run.py @@ -14,7 +14,7 @@ from __future__ import annotations -from collections.abc import Iterable, Sequence +from collections.abc import Iterable, Iterator, Sequence from dataclasses import dataclass from typing import TYPE_CHECKING, Any @@ -48,23 +48,55 @@ def depends_on_job_nodes(node: Any) -> bool: return any(_is_job_node(upstream) for upstream in node.upstreams) +def _streams_read_by(node: Any) -> Iterator[Any]: + """Yield the streams outside a job that a run of ``node`` reads. + + The node's inputs, the stream bound to a source slot among them, and, through every + stream outside a job, that stream's own inputs; a job node other than a source slot + is not entered (the run reads its held rows or runs it as a node of its own). + + Args: + node: A job node; for a source slot, its bound stream is what it reads. + """ + from orcapod.core.nodes.source_node import SourceJobNode + + pending: list[Any] = ( + [node.bound_source] if isinstance(node, SourceJobNode) else list(node.upstreams) + ) + seen: list[Any] = [] + while pending: + stream = pending.pop() + if stream is None or any(stream is other for other in seen): + continue + seen.append(stream) + if isinstance(stream, SourceJobNode): + pending.append(stream.bound_source) + continue + if _is_job_node(stream): + continue + yield stream + pending.extend(getattr(stream, "upstreams", ())) + + def refuse_delivering_inputs(node: Any) -> None: """Refuse a load-only run of ``node`` over an input that delivers when it is read. A load-only run computes, delivers and writes nothing (ITL-1095). The stream a side-effect pod returns outside a job (``SideEffectPodStream``) delivers its side - effects on every read, so a load-only run cannot read it; it is refused before - anything is read (ITL-1121). Another stream outside a job is read as it always is. + effects on every read, so a load-only run cannot read it, directly, through a + source slot bound to it, or through a stream outside a job that reads it; it is + refused before anything is read (ITL-1121). Another stream outside a job is read + as it always is. Args: - node: A job node a load-only run would run. + node: A job node a load-only run would run or read. Raises: - ValueError: If an input of ``node`` is a ``SideEffectPodStream``. + ValueError: If a run of ``node`` would read a ``SideEffectPodStream``. """ from orcapod.side_effects import SideEffectPodStream - for upstream in node.upstreams: + for upstream in _streams_read_by(node): if isinstance(upstream, SideEffectPodStream): raise ValueError( f"A load-only run cannot run node {node.label!r}: its input " diff --git a/src/orcapod/side_effects.py b/src/orcapod/side_effects.py index e9c9a7cf9..2fe4cac96 100644 --- a/src/orcapod/side_effects.py +++ b/src/orcapod/side_effects.py @@ -1275,6 +1275,11 @@ def _run_alone(self, *, load_only: bool) -> None: except Exception as exc: self.clear_cache() raise StoreWriteError(database.base_path, flush=True) from exc + except BaseException: + # An interruption leaves the commit's outcome unknown: the node holds no + # run, so the next run runs and commits again (ITL-1121). + self.clear_cache() + raise # ------------------------------------------------------------------ # Async execution diff --git a/superpowers/scripts/itl_1121/c3_node_run_pending_writes.py b/superpowers/scripts/itl_1121/c3_node_run_pending_writes.py index 2cce0f305..d6cb7fabc 100644 --- a/superpowers/scripts/itl_1121/c3_node_run_pending_writes.py +++ b/superpowers/scripts/itl_1121/c3_node_run_pending_writes.py @@ -9,6 +9,8 @@ import tempfile from pathlib import Path +from typing import Any + import deltalake import pyarrow as pa @@ -29,36 +31,49 @@ def col(table: deltalake.DeltaTable, name: str) -> set[object]: ) +# Every Delta commit, in the order it happened: (table directory, rows written). Commit +# timestamps have millisecond resolution, so equal timestamps could not order two commits; +# the order of the write calls does. +COMMITS: list[tuple[str, pa.Table]] = [] +_real_write = deltalake.write_deltalake + + +def _recording_write(table_or_uri: Any, data: Any, *args: Any, **kwargs: Any) -> Any: + COMMITS.append( + (str(table_or_uri), pa.table(data) if not isinstance(data, pa.Table) else data) + ) + return _real_write(table_or_uri, data, *args, **kwargs) + + +deltalake.write_deltalake = _recording_write # pyright: ignore[reportAttributeAccessIssue] # a recording wrapper for this script only + + def check(root: Path) -> None: + pipeline = [ + t for path, t in COMMITS if "_result" not in path and "leftover" not in path + ] + results = [t for path, t in COMMITS if "_result" in path] tables = { ("result" if "_result" in t.relative_to(root).parts else "pipeline"): t for t in {log.parent for log in root.rglob("_delta_log")} if t.name != "leftover" } - result_history = deltalake.DeltaTable(str(tables["result"])).history() - pipeline_history = deltalake.DeltaTable(str(tables["pipeline"])).history() + history = { + kind: len(deltalake.DeltaTable(str(t)).history()) for kind, t in tables.items() + } print( - f" pipeline commits: {len(pipeline_history)}, result commits: {len(result_history)}" + f" recorded commits: pipeline {len(pipeline)}, result {len(results)}; " + f"table histories: {history}" ) + committed: set[object] = set() bad = 0 - for entry in pipeline_history: - records = deltalake.DeltaTable( - str(tables["pipeline"]), version=entry["version"] - ) - versions = [ - r["version"] for r in result_history if r["timestamp"] <= entry["timestamp"] - ] - committed = ( - col( - deltalake.DeltaTable(str(tables["result"]), version=max(versions)), - "__record_id", - ) - if versions - else set() - ) - if not col(records, "__data_id") <= committed: - bad += 1 - print(f" pipeline versions referencing uncommitted results: {bad}") + for path, table in COMMITS: + if "_result" in path: + committed |= set(table.column("__record_id").to_pylist()) + elif "leftover" not in path: + if not set(table.column("__data_id").to_pylist()) <= committed: + bad += 1 + print(f" pipeline commits referencing a result not committed before them: {bad}") for batch_size, n in [(50, 120), (5, 40)]: @@ -81,6 +96,7 @@ def check(root: Path) -> None: ), label="dbl", ) + COMMITS.clear() print(f"batch_size={batch_size}, rows={n}: node.run()") job.nodes["dbl"].run() check(root) diff --git a/superpowers/scripts/itl_1121/c9_interrupted_log_commit.py b/superpowers/scripts/itl_1121/c9_interrupted_log_commit.py new file mode 100644 index 000000000..66f879930 --- /dev/null +++ b/superpowers/scripts/itl_1121/c9_interrupted_log_commit.py @@ -0,0 +1,57 @@ +"""PR #381 summary finding: a standalone side-effect job node (input outside a job, store +attached) holds its rows before _run_alone commits the invocation log. A KeyboardInterrupt +during that commit skips the Exception-only cleanup, so the node keeps a current run: the +next run() is held and never retries the commit.""" + +from typing import Any +from unittest import mock + +import pyarrow as pa + +from orcapod.core.streams import ArrowTableStream +from orcapod.databases import InMemoryArrowDatabase +from orcapod.side_effects import SideEffectJobNode, tap_pod + +DELIVERED: list[int] = [] + + +def deliver(x: int, ctx: Any) -> None: + DELIVERED.append(x) + + +schema = pa.schema( + [ + pa.field("k", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] +) +stream = ArrowTableStream( + pa.table({"k": [1, 2], "x": [1, 2]}, schema=schema), tag_columns=["k"] +) +db = InMemoryArrowDatabase() +tap: Any = tap_pod(deliver) +node = SideEffectJobNode(tap, stream) +node.attach_databases(db) + +with mock.patch.object(InMemoryArrowDatabase, "flush", side_effect=KeyboardInterrupt): + try: + node.run() + except KeyboardInterrupt: + print("run(): KeyboardInterrupt during the log commit") +print( + f"after the interrupted commit: holds_current_run={node.holds_current_run(load_only=False)} is_stale={node.is_stale} delivered={len(DELIVERED)}" +) +flushes: list[int] = [] +real_flush = InMemoryArrowDatabase.flush + + +def counting_flush(self: Any) -> None: + flushes.append(1) + real_flush(self) + + +with mock.patch.object(InMemoryArrowDatabase, "flush", counting_flush): + node.run() +print( + f"next run(): commits attempted={len(flushes)} delivered in total={len(DELIVERED)}" +) diff --git a/tests/test_pipeline/test_node_view_review_fixes.py b/tests/test_pipeline/test_node_view_review_fixes.py index 8135f9682..24f70ff2c 100644 --- a/tests/test_pipeline/test_node_view_review_fixes.py +++ b/tests/test_pipeline/test_node_view_review_fixes.py @@ -404,3 +404,90 @@ def test_a_full_run_over_an_input_that_delivers_reads_it(kind: str) -> None: node = _node_over(kind, _delivering_stream()) assert node.run().as_table().num_rows == 2 assert len(_UPSTREAM_DELIVERED) == 2 + + +def _job_over_source_bound_to_a_delivering_stream() -> PipelineJob: + """``tw`` (``_twice``) over source slot ``s``, run once, then bound to a stream that + delivers when read.""" + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + source = DictSource( + [{"id": 1, "x": 1}, {"id": 2, "x": 2}], tag_columns=["id"], source_id="s" + ) + _twice_pod(source, label="tw") + job.run() + job.bind(sources={"s": _delivering_stream()}) + return job + + +@pytest.mark.parametrize("how", ["sync", "async", "node.run", "source.run"]) +def test_a_load_only_run_refuses_a_source_slot_bound_to_a_delivering_stream( + how: str, +) -> None: + """A source slot reads its bound stream: refused before it is read.""" + _UPSTREAM_DELIVERED.clear() + job = _job_over_source_bound_to_a_delivering_stream() + with pytest.raises(ValueError, match="delivers its side effects when it is read"): + if how == "node.run": + job.nodes["tw"].run(load_only=True) + elif how == "source.run": + job.nodes["tw"].upstreams[0].run(load_only=True) + else: + job.run(orchestrator=ORCHESTRATORS[how](), load_only=True) + assert _UPSTREAM_DELIVERED == [] + + +@pytest.mark.parametrize("kind", ["side_effect", "function", "operator"]) +def test_a_load_only_run_refuses_a_stream_that_reads_a_delivering_stream( + kind: str, +) -> None: + """An operator's stream outside a job over a side-effect stream delivers when read.""" + _UPSTREAM_DELIVERED.clear() + stream: Any = _delivering_stream().select_data_columns(["x"]) + node = _node_over(kind, stream) + with pytest.raises(ValueError, match="delivers its side effects when it is read"): + node.run(load_only=True) + assert _UPSTREAM_DELIVERED == [] + + +def test_a_load_only_operator_run_without_a_database_is_loaded_only() -> None: + """Its rows come from a load-only run: shown as such, and not a full run's result.""" + from orcapod.core.nodes.operator_node import OperatorJobNode + from orcapod.core.operators import MapData + from orcapod.core.streams import ViewStatus + + node = OperatorJobNode( + operator=MapData({"x": "y"}), + input_streams=(_base_stream(),), + pipeline_database=None, + ) + assert node.run(load_only=True).status == ViewStatus.LOADED_ONLY + assert node.holds_current_run(load_only=True) + assert not node.holds_current_run(load_only=False) + assert node.run().status == ViewStatus.ROWS + + +def test_an_interrupted_log_commit_leaves_the_side_effect_node_without_a_run() -> None: + """A standalone side-effect node with a store: a KeyboardInterrupt while its run commits + the invocation log clears the node, so the next run runs (and commits) again.""" + from unittest import mock + + from orcapod.side_effects import SideEffectJobNode + + _DELIVERED.clear() + node = SideEffectJobNode(_tap(_deliver_own), _base_stream()) + node.attach_databases(InMemoryArrowDatabase()) + with ( + mock.patch.object( + InMemoryArrowDatabase, "flush", side_effect=KeyboardInterrupt + ), + pytest.raises(KeyboardInterrupt), + ): + node.run() + assert not node.holds_current_run(load_only=False) + assert node.is_stale + with mock.patch.object( + InMemoryArrowDatabase, "flush", autospec=True, side_effect=lambda self: None + ) as flush: + node.run() + assert flush.call_count >= 1 From 6fe3c6e6526d75f43f1022ace9c62c22aa169b96 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 09:26:23 -0800 Subject: [PATCH 52/63] chore(repro): a held node.run() keeps the previous last_run (observation, ITL-1121) Co-Authored-By: Claude Opus 5.5 --- .../c10_held_node_run_keeps_last_run.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 superpowers/scripts/itl_1121/c10_held_node_run_keeps_last_run.py diff --git a/superpowers/scripts/itl_1121/c10_held_node_run_keeps_last_run.py b/superpowers/scripts/itl_1121/c10_held_node_run_keeps_last_run.py new file mode 100644 index 000000000..b34049405 --- /dev/null +++ b/superpowers/scripts/itl_1121/c10_held_node_run_keeps_last_run.py @@ -0,0 +1,25 @@ +"""Observation (not changed): node.run() of a node that holds a current run returns at once +and makes no summary, so node.last_run stays the summary of the run before it.""" + +from orcapod import PipelineJob, function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.sources import DictSource + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + return x + 1 + + +job = PipelineJob(name="j", database=InMemoryArrowDatabase()) +with job: + inc.pod( + DictSource([{"k": 1, "x": 1}], tag_columns=["k"], source_id="s"), + label="rowwise", + ) +node = job.nodes["rowwise"] +node.run() +first = node.last_run +node.run() # held: runs nothing +print(f"second run() left last_run unchanged: {node.last_run is first}") +print(repr(node.last_run).splitlines()[0]) From d06ffcf7a5dd17b9df26d3a6e0050c665357fb3a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 09:48:26 -0800 Subject: [PATCH 53/63] feat(node_run): a load-only run refuses an input that computes when read (ITL-1121) Developer decision (November 14, 2026, 'Refuse is consistent'): a load-only run reads only job nodes, sources and streams that neither compute nor deliver. A function pod's stream outside a job (a cached pod's included) is now refused before anything is read, as a side-effect pod's stream already was: as a node's input, bound to a source slot, or read through an operator's stream outside the job; on job.run, run_async, node.run and a source slot's run. One rule, node_run.refuse_unloadable_inputs. Docs say what to do instead. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1121.md | 5 +- design_issues/ITL-1121-NV1.md | 8 +- docs/concepts/function-pods.md | 7 ++ src/orcapod/core/nodes/function_node.py | 4 +- src/orcapod/core/nodes/operator_node.py | 4 +- src/orcapod/core/nodes/source_node.py | 14 +-- src/orcapod/pipeline/node_run.py | 55 ++++++---- .../c6_load_only_over_side_effect_stream.py | 17 +-- .../test_node_view_review_fixes.py | 102 ++++++++++++++++++ 9 files changed, 174 insertions(+), 42 deletions(-) diff --git a/changelog.d/ITL-1121.md b/changelog.d/ITL-1121.md index 128ac6113..7d0217086 100644 --- a/changelog.d/ITL-1121.md +++ b/changelog.d/ITL-1121.md @@ -1,9 +1,12 @@ +### Changed + +- A load-only run (`job.run(load_only=True)`, `run_async(load_only=True)`, `node.run(load_only=True)`, a source slot's `run(load_only=True)`) reads only job nodes, sources and streams that neither compute nor deliver (developer, November 14, 2026; ITL-1121). A pod's stream built outside the job is refused with a `ValueError` before anything is read: a function pod's stream (a cached pod's included), which computes when it is read, and a side-effect pod's stream, which delivers its side effects (before, the run read it and delivered every row). This applies whether a job node reads the stream, a source slot is bound to it or an operator's stream outside the job reads it. Build the pod inside the job, or run without `load_only`; a run without `load_only` is unchanged. + ### Fixed - An observer that read a function job node in `on_node_end` emptied it: the node was marked current only after that hook, so the read found it stale and cleared the run's rows (ITL-1121). The node is now current before the hook, on sync, async and load-only runs. - An operator job node without a database, or in `CacheMode.OFF`, whose output failed to compute in `run()` kept claiming a current run and its reads raised; it now reads as not run, like every node whose run raised. - The run summary said "nothing to do; every node holds a current run" when the run ran a side-effect node (and delivered) or an operator node below a held function node. It now says so only when the run ran no node; `RunSummary` takes `ran_nothing`. - A side-effect job node whose load-only `execute()` failed on its input stayed current, and so did the nodes below it; it is now cleared, as on the async path. -- A load-only run delivered side effects when it read a side-effect pod's stream outside a job: as the input of a job node (`node.run(load_only=True)`), through another stream outside a job that reads it, or bound to a job's source slot (`job.run(load_only=True)`, `run_async`, a node's or the source slot's own `run(load_only=True)`). A load-only run now refuses such an input with a `ValueError` before anything is read; a run without `load_only` is unchanged. - An operator job node without a database, run with `load_only=True`, showed its rows as a full run's (`rows` instead of `loaded_only`) and a later full `run()` took them as its own. - A side-effect job node over a stream outside a job, with a store attached, kept its rows as a current run when its `run()` was interrupted (`KeyboardInterrupt`) while committing the invocation log, so the next `run()` neither ran nor committed again; it is now cleared. diff --git a/design_issues/ITL-1121-NV1.md b/design_issues/ITL-1121-NV1.md index 3f2072ae8..85b0a8d50 100644 --- a/design_issues/ITL-1121-NV1.md +++ b/design_issues/ITL-1121-NV1.md @@ -21,7 +21,8 @@ through the public interface (`superpowers/scripts/itl_1121/`): time, so it and the nodes below it were not stale. - A load-only run read a side-effect pod's stream outside a job (directly, through another stream outside a job, or bound to a source slot), which delivered every row, although a - load-only run delivers nothing. + load-only run delivers nothing; it also read a function pod's stream outside a job, which + computed. - An operator job node without a database did not record a load-only run as one. - A standalone side-effect job node interrupted while committing its invocation log kept a current run, so the next run neither ran nor committed. @@ -29,8 +30,9 @@ through the public interface (`superpowers/scripts/itl_1121/`): **Fix:** stamp before `on_node_end`; build the computed view inside the clearing block; `RunSummary(ran_nothing=...)` from the run plan; clear the node when the load-only input raises; the run plan and a source slot's `run()` refuse (`ValueError`, before reading) a -load-only run that would read a `SideEffectPodStream` (`node_run.refuse_delivering_inputs`, -walking streams outside a job and bound source slots); the no-database operator branch +load-only run that would read a `SideEffectPodStream` or a `FunctionPodStream` +(`node_run.refuse_unloadable_inputs`, walking streams outside a job and bound source slots; +developer, November 14, 2026: "Refuse is consistent"); the no-database operator branch records `load_only`; the side-effect node's log commit clears the node on any interruption. Two review comments did not reproduce or are tracked elsewhere: a node run outside `job.run()` commits every row, so no pre-run flush is needed; a function node with no database holding no rows after an async run is diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index a0c53235b..76aae9310 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -577,6 +577,13 @@ incomplete: a group formed after it may lack members, and the result stored for group by an earlier run may then be loaded for it. `run_async(load_only=True)` does the same on the asynchronous orchestrator. +A load-only run reads only job nodes, sources and streams that neither compute nor deliver. +A pod's stream built outside the job (a function pod's, which computes when it is read, or a +side-effect pod's, which delivers) is refused with a `ValueError` before anything is read, +whether a node of the job reads it, a source slot is bound to it or an operator's stream +outside the job reads it. Build the pod inside the job, so its node loads its stored results, +or run without `load_only`. + ### Running again: what a run runs `job.run()`, `await job.run_async()` and `job.nodes[label].run()` decide what to run from diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index f3cb27c04..fb7cabbaf 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -302,7 +302,7 @@ def run_job_node( """ from orcapod.pipeline.node_run import ( depends_on_job_nodes, - refuse_delivering_inputs, + refuse_unloadable_inputs, run_with_dependencies, ) from orcapod.pipeline.run_summary import warn_row_failures @@ -313,7 +313,7 @@ def run_job_node( summary = run_with_dependencies(node, load_only=load_only) else: if load_only: - refuse_delivering_inputs(node) + refuse_unloadable_inputs(node) summary = run_alone() if summary is None: return diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index e4f08be20..62e7400d8 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -1531,7 +1531,7 @@ def run(self, *, load_only: bool = False) -> StreamView: """ from orcapod.pipeline.node_run import ( depends_on_job_nodes, - refuse_delivering_inputs, + refuse_unloadable_inputs, run_with_dependencies, ) from orcapod.pipeline.run_summary import warn_row_failures @@ -1549,7 +1549,7 @@ def run(self, *, load_only: bool = False) -> StreamView: if self._loaded: raise self._cannot_compute_error() if load_only: - refuse_delivering_inputs(self) + refuse_unloadable_inputs(self) if self.is_stale or not self._holds_run(load_only=load_only): # Stale, or a load-only run's output asked for by a full run: run again. self.clear_cache() diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index f4ac4d8d8..c0309179f 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -594,22 +594,22 @@ def __init__( def run(self, *, load_only: bool = False) -> StreamView: """Read the bound source's rows and return a ``StreamView`` of all of them. - A source is read, never computed, so ``load_only`` changes what is read only - for a bound stream that delivers side effects when read, which a load-only run - refuses (ITL-1121); a job node's ``run()`` takes it (ITL-1095). + A source is read, never computed, so ``load_only`` changes nothing, except that + a load-only run refuses a bound stream that computes or delivers when read + (ITL-1121); a job node's ``run()`` takes it (ITL-1095). Args: load_only: Accepted for every job node; a source reads its rows either way. Raises: UnboundSourceError: When no concrete source is bound. - ValueError: If ``load_only`` and the bound stream delivers side effects - when read (``node_run.refuse_delivering_inputs``). + ValueError: If ``load_only`` and the bound stream computes or delivers + when read (``node_run.refuse_unloadable_inputs``). """ if load_only: - from orcapod.pipeline.node_run import refuse_delivering_inputs + from orcapod.pipeline.node_run import refuse_unloadable_inputs - refuse_delivering_inputs(self) + refuse_unloadable_inputs(self) return super().run() @property diff --git a/src/orcapod/pipeline/node_run.py b/src/orcapod/pipeline/node_run.py index 48c5125de..3efb70f31 100644 --- a/src/orcapod/pipeline/node_run.py +++ b/src/orcapod/pipeline/node_run.py @@ -78,33 +78,46 @@ def _streams_read_by(node: Any) -> Iterator[Any]: pending.extend(getattr(stream, "upstreams", ())) -def refuse_delivering_inputs(node: Any) -> None: - """Refuse a load-only run of ``node`` over an input that delivers when it is read. - - A load-only run computes, delivers and writes nothing (ITL-1095). The stream a - side-effect pod returns outside a job (``SideEffectPodStream``) delivers its side - effects on every read, so a load-only run cannot read it, directly, through a - source slot bound to it, or through a stream outside a job that reads it; it is - refused before anything is read (ITL-1121). Another stream outside a job is read - as it always is. +def refuse_unloadable_inputs(node: Any) -> None: + """Refuse a load-only run of ``node`` that would read an input which computes or + delivers when it is read. + + A load-only run computes, delivers and writes nothing (ITL-1095), so it reads only + job nodes, sources and streams that neither compute nor deliver (developer, + November 14, 2026: "Refuse is consistent"; ITL-1121). Outside a job, a function + pod's stream (``FunctionPodStream``, a cached pod's included) computes its rows when + read, and a side-effect pod's stream (``SideEffectPodStream``) delivers its side + effects. Such a stream is refused before anything is read, whether it is an input of + ``node``, bound to a source slot, or read through another stream outside a job (an + operator's stream over it). Args: node: A job node a load-only run would run or read. Raises: - ValueError: If a run of ``node`` would read a ``SideEffectPodStream``. + ValueError: If a run of ``node`` would read such a stream. """ + from orcapod.core.function_pod import FunctionPodStream from orcapod.side_effects import SideEffectPodStream for upstream in _streams_read_by(node): if isinstance(upstream, SideEffectPodStream): - raise ValueError( - f"A load-only run cannot run node {node.label!r}: its input " - f"{upstream.label!r} is a side-effect pod's stream outside a job, which " - "delivers its side effects when it is read, and a load-only run " - "delivers nothing. Run it without load_only, or build the side-effect " - "pod inside the job." + what = ( + "a side-effect pod's stream outside a job, which delivers its side " + "effects when it is read, and a load-only run delivers nothing" + ) + elif isinstance(upstream, FunctionPodStream): + what = ( + "a function pod's stream outside a job, which computes when it is " + "read, and a load-only run computes nothing" ) + else: + continue + raise ValueError( + f"A load-only run cannot run node {node.label!r}: its input " + f"{upstream.label!r} is {what}. Build the pod inside the job, or run " + "without load_only." + ) def _stores(nodes: list[Any]) -> list[ArrowDatabaseProtocol]: @@ -174,7 +187,7 @@ def plan_run(graph: Any, *, load_only: bool, outside: Sequence[Any] = ()) -> Run (``outside``) is read for the nodes it feeds; every other node runs. The same plan serves ``job.run()`` (its whole graph) and ``node.run()`` (the node and what it depends on). A load-only plan refuses, before reading anything, a node to run whose - input delivers when read (``refuse_delivering_inputs``). + input computes or delivers when read (``refuse_unloadable_inputs``). Args: graph: The run's node graph (``topological_sort``, ``predecessors``). @@ -186,7 +199,7 @@ def plan_run(graph: Any, *, load_only: bool, outside: Sequence[Any] = ()) -> Run The plan. Raises: - ValueError: As ``refuse_delivering_inputs``, for a load-only run. + ValueError: As ``refuse_unloadable_inputs``, for a load-only run. """ from orcapod.core.nodes.source_node import SourceJobNode from orcapod.core.nodes.staleness import RunHoldingNode @@ -219,9 +232,9 @@ def is_source(node: Any) -> bool: ] if load_only: # Refused before any stream is read below: a load-only run cannot read an - # input that delivers (ITL-1121). + # input that computes or delivers (ITL-1121). for node in to_run: - refuse_delivering_inputs(node) + refuse_unloadable_inputs(node) included: list[Any] = [] for node in order: feeds_a_run = any( @@ -291,7 +304,7 @@ def run_with_dependencies(target: Any, *, load_only: bool) -> RunSummary: Raises: StoreWriteError: If committing the run's writes failed. ValueError: For a load-only run, if a node it would run reads an input that - delivers when read (``refuse_delivering_inputs``); nothing has run then. + computes or delivers when read (``refuse_unloadable_inputs``); nothing has run then. """ from orcapod.core.nodes.source_node import SourceJobNode from orcapod.pipeline.dag import OrcaDAG diff --git a/superpowers/scripts/itl_1121/c6_load_only_over_side_effect_stream.py b/superpowers/scripts/itl_1121/c6_load_only_over_side_effect_stream.py index c2390a5e1..f376e30f1 100644 --- a/superpowers/scripts/itl_1121/c6_load_only_over_side_effect_stream.py +++ b/superpowers/scripts/itl_1121/c6_load_only_over_side_effect_stream.py @@ -63,12 +63,17 @@ def dbl(y: int) -> int: print(f"side-effect node, run(load_only=True) refused: {exc}") print(f" upstream delivered {len(UP)}, own delivered {len(OWN)}") -# Unchanged (reported to the coordinator): an outside FunctionPodStream is read, so a -# load-only run computes its function. +# A function pod's stream outside a job computes when read. Before the developer's decision +# (November 14, 2026, "Refuse is consistent") a load-only run read it and called the +# function; since then it is refused as well. fn_upstream: Any = inc.pod(base) # a FunctionPodStream, outside a job dbl_pod: Any = dbl.pod fnode = FunctionJobNode(dbl_pod, fn_upstream, pipeline_database=InMemoryArrowDatabase()) -fview = fnode.run(load_only=True) -print( - f"function node over a pod stream, run(load_only=True): upstream function called {len(CALLS)} times; status {fview.status}" -) +try: + fnode.run(load_only=True) + print( + f"function node over a pod stream, run(load_only=True): upstream calls {len(CALLS)}" + ) +except ValueError as exc: + print(f"function node over a pod stream, run(load_only=True) refused: {exc}") +print(f" upstream function calls: {len(CALLS)}") diff --git a/tests/test_pipeline/test_node_view_review_fixes.py b/tests/test_pipeline/test_node_view_review_fixes.py index 24f70ff2c..631c49a9e 100644 --- a/tests/test_pipeline/test_node_view_review_fixes.py +++ b/tests/test_pipeline/test_node_view_review_fixes.py @@ -491,3 +491,105 @@ def test_an_interrupted_log_commit_leaves_the_side_effect_node_without_a_run() - ) as flush: node.run() assert flush.call_count >= 1 + + +# --------------------------------------------------------------------------- +# Developer decision (November 14, 2026): a load-only run also refuses an input that +# computes when read ("Refuse is consistent") +# --------------------------------------------------------------------------- + +_UPSTREAM_CALLS: list[int] = [] +_COMPUTES = "computes when it is read" + + +@function_pod(output_keys="x") +def _recorded_copy(x: int) -> int: + """Return ``x``, recording the call.""" + _UPSTREAM_CALLS.append(x) + return x + + +def _computing_stream(kind: str) -> Any: + """A stream outside a job that computes ``_recorded_copy`` when read. + + ``pod``: the function pod's stream; ``cached``: a cached function pod's stream over + an empty store; ``wrapped``: an operator's stream outside a job over the pod's + stream. (A ``FunctionNode`` outside a job is a blueprint: reading it raises.) + """ + from orcapod.core.cached_function_pod import CachedFunctionPod + + base = _base_stream() + pod: Any = _recorded_copy.pod + if kind == "pod": + return pod(base) + if kind == "cached": + return CachedFunctionPod(pod, result_database=InMemoryArrowDatabase())(base) + stream: Any = pod(base) + return stream.select_data_columns(["x"]) + + +@pytest.mark.parametrize("stream_kind", ["pod", "cached", "wrapped"]) +@pytest.mark.parametrize("kind", ["side_effect", "function", "operator"]) +def test_a_load_only_run_refuses_an_input_that_computes( + kind: str, stream_kind: str +) -> None: + """Refused before the input is read: its function is never called.""" + _UPSTREAM_CALLS.clear() + node = _node_over(kind, _computing_stream(stream_kind)) + with pytest.raises(ValueError, match=_COMPUTES): + node.run(load_only=True) + assert _UPSTREAM_CALLS == [] + assert not node.holds_current_run(load_only=True) + + +@pytest.mark.parametrize("kind", ["side_effect", "function", "operator"]) +def test_a_load_only_cascade_refuses_an_input_that_computes(kind: str) -> None: + """The computing input feeds a node the cascade would run first.""" + from orcapod.core.nodes.operator_node import OperatorJobNode + from orcapod.core.operators import DropNulls + from orcapod.types import CacheMode + + _UPSTREAM_CALLS.clear() + first = _node_over(kind, _computing_stream("pod")) + target = OperatorJobNode( + operator=DropNulls(), + input_streams=(first,), + pipeline_database=InMemoryArrowDatabase(), + cache_mode=CacheMode.OFF, + ) + with pytest.raises(ValueError, match=_COMPUTES): + target.run(load_only=True) + assert _UPSTREAM_CALLS == [] + + +@pytest.mark.parametrize("how", ["sync", "async", "node.run", "source.run"]) +def test_a_load_only_run_refuses_a_source_slot_bound_to_a_computing_stream( + how: str, +) -> None: + """A source slot bound to a function pod's stream outside a job: refused.""" + _UPSTREAM_CALLS.clear() + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + source = DictSource( + [{"id": 1, "x": 1}, {"id": 2, "x": 2}], tag_columns=["id"], source_id="s" + ) + _twice_pod(source, label="tw") + job.run() + job.bind(sources={"s": _computing_stream("pod")}) + with pytest.raises(ValueError, match=_COMPUTES): + if how == "node.run": + job.nodes["tw"].run(load_only=True) + elif how == "source.run": + job.nodes["tw"].upstreams[0].run(load_only=True) + else: + job.run(orchestrator=ORCHESTRATORS[how](), load_only=True) + assert _UPSTREAM_CALLS == [] + + +@pytest.mark.parametrize("kind", ["side_effect", "function", "operator"]) +def test_a_full_run_over_an_input_that_computes_reads_it(kind: str) -> None: + """Unchanged: a run without ``load_only`` reads (computes) the outside stream.""" + _UPSTREAM_CALLS.clear() + node = _node_over(kind, _computing_stream("pod")) + assert node.run().as_table().num_rows == 2 + assert len(_UPSTREAM_CALLS) == 2 From ac6335438d790ecbb303d4736fff88c3c2cb5037 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 09:49:13 -0800 Subject: [PATCH 54/63] docs(plan): ITL-1121 progress notes Co-Authored-By: Claude Opus 5.5 --- .../plans/2026-11-14-itl-1121-node-view-review-fixes.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md b/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md index 0d364d2e1..43607b34a 100644 --- a/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md +++ b/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md @@ -148,3 +148,9 @@ Reproduction scripts: `superpowers/scripts/itl_1121/c1..c6_*.py` (committed). full `uv run pytest tests/` + test-objective, draft PR (top line "Stacked on #380: merge after it. Fixes ITL-1121"), orca linear attach / worktree set --pr, signed replies on the nine #381 findings (3: evidence c3; 7: ITL-1097), CI green, CodeRabbit round, worker_done. +- 2026-11-14 (resumed): changelog, design issue, draft PR #388, #381 replies (6 threads + 1 top-level), + CodeRabbit round 1 on #388 fixed (C, D/F, H, I; 75324abc) and answered (A, B, E, G, J, K, L), + KeyboardInterrupt log-commit fix (c9). Developer decision: a load-only run also refuses a + computing outside stream (d06ffcf7, node_run.refuse_unloadable_inputs). main merged (#382, #383). + LEFT: name the coordinator's follow-up issue in the reply to finding G (thread 4574191943); + CI and a CodeRabbit review of the head; worker_done. From 5dd8d6a315b90bbf2e92067a764b99cf0e8770c3 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 09:49:51 -0800 Subject: [PATCH 55/63] docs(function_pods): name ITL-1130, which re-evaluates the load-only refusal (ITL-1121) Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1121.md | 2 +- docs/concepts/function-pods.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/changelog.d/ITL-1121.md b/changelog.d/ITL-1121.md index 7d0217086..c4f5ae9d3 100644 --- a/changelog.d/ITL-1121.md +++ b/changelog.d/ITL-1121.md @@ -1,6 +1,6 @@ ### Changed -- A load-only run (`job.run(load_only=True)`, `run_async(load_only=True)`, `node.run(load_only=True)`, a source slot's `run(load_only=True)`) reads only job nodes, sources and streams that neither compute nor deliver (developer, November 14, 2026; ITL-1121). A pod's stream built outside the job is refused with a `ValueError` before anything is read: a function pod's stream (a cached pod's included), which computes when it is read, and a side-effect pod's stream, which delivers its side effects (before, the run read it and delivered every row). This applies whether a job node reads the stream, a source slot is bound to it or an operator's stream outside the job reads it. Build the pod inside the job, or run without `load_only`; a run without `load_only` is unchanged. +- A load-only run (`job.run(load_only=True)`, `run_async(load_only=True)`, `node.run(load_only=True)`, a source slot's `run(load_only=True)`) reads only job nodes, sources and streams that neither compute nor deliver (developer, November 14, 2026; ITL-1121). A pod's stream built outside the job is refused with a `ValueError` before anything is read: a function pod's stream (a cached pod's included), which computes when it is read, and a side-effect pod's stream, which delivers its side effects (before, the run read it and delivered every row). This applies whether a job node reads the stream, a source slot is bound to it or an operator's stream outside the job reads it. Build the pod inside the job, or run without `load_only`; a run without `load_only` is unchanged. ITL-1130 re-evaluates the choice. ### Fixed diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 76aae9310..cf691ea5d 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -582,7 +582,8 @@ A pod's stream built outside the job (a function pod's, which computes when it i side-effect pod's, which delivers) is refused with a `ValueError` before anything is read, whether a node of the job reads it, a source slot is bound to it or an operator's stream outside the job reads it. Build the pod inside the job, so its node loads its stored results, -or run without `load_only`. +or run without `load_only`. (Refusing is the consistent choice for now; ITL-1130 re-evaluates +it.) ### Running again: what a run runs From 5a960d108a8fa8c70d52b6926e78fde118a38c1f Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 10:34:09 -0800 Subject: [PATCH 56/63] test(pipeline): a job node's current-run view has one Arrow schema in every state (ITL-1095) Function, group_by and join nodes over an int8 non-null tag and int32 data: as_table(all_info=True) has the populated table's schema before a run, after a run, after clear_cache, after a failed run, while stale and after a load-only run, sync and async. A join that receives no rows lists its columns in another order (pre-existing, ITL-1131): strict xfail. Co-Authored-By: Claude Opus 5.5 --- .../2026-11-14-itl-1095-merge-main-itl-912.md | 50 +++++ .../test_current_run_view_zero_row_types.py | 198 ++++++++++++++++++ 2 files changed, 248 insertions(+) create mode 100644 superpowers/plans/2026-11-14-itl-1095-merge-main-itl-912.md create mode 100644 tests/test_pipeline/test_current_run_view_zero_row_types.py diff --git a/superpowers/plans/2026-11-14-itl-1095-merge-main-itl-912.md b/superpowers/plans/2026-11-14-itl-1095-merge-main-itl-912.md new file mode 100644 index 000000000..cd93664f1 --- /dev/null +++ b/superpowers/plans/2026-11-14-itl-1095-merge-main-itl-912.md @@ -0,0 +1,50 @@ +# ITL-1095: merge main (with #375, ITL-912) into PR #380 + +Coordinator task task_841fb1367e00, 2026-11-14. Branch `eywalker/itl-1095-design-what-a-job-node-shows-the-current`, +head before merge `7de3512b`, main `52d67858`. + +## Goal + +Both rules hold after the merge: + +- #380 (ITL-1095): a job node shows its current run; before a run, after `clear_cache()`, after a + failed run and while stale it reads zero rows and touches no store; `LOADED_ONLY`; state-driven runs. +- #375 (ITL-912): that zero-row table has the Arrow schema of the node's populated table (types, + nullability, system tags, source info, column order), derived from input schemas only. + +## Conflicts and resolution + +| File | Resolution | +|---|---| +| `CLAUDE.md`, `.zed/rules` | main's head() paragraph (ITL-991, ITL-912), last sentence replaced by #380's `load_only` rule | +| `core/nodes/function_node.py` | both: #380's `_read_finds_stale` and main's `_zero_row_output_table` / `_typed_zero_row_table` / `_zero_row_table`; the empty current-run table already goes through `_table_from_pairs([])`, which main types | +| `core/nodes/operator_node.py` | `as_table` docstring: #380's current-run text + main's typed zero-row paragraph (REPLAY store read and "loaded nodes raise" dropped: #380 removed them). Body is #380's: `_make_empty_table()`, which main made typed. `head` summary: #380's | +| `tests/.../test_loaded_operator_cache.py` | helper keeps #380's stored-view comparison (`as_source()`) and main's `same_types` parameter | + +## Test changes (ask coordinator before push) + +- `test_an_absent_cache_table_gives_the_running_nodes_types` (#375, strict xfail ITL-1089): drop the + `pytest.warns(StoreContentWarning)` wrapper; under #380 `as_table()` of a node that has not run + never reads the store, so it never warns. The schema comparison (the xfail's point) is unchanged. + +## New test + +`tests/test_pipeline/test_current_run_view_zero_row_types.py`: function node and operator nodes +(group_by, join) over an int8 not-null tag / int32 data source; `as_table(all_info=True).schema` is the +same before a run, after a run, after `clear_cache()`, after a failed run, stale, after +`run(load_only=True)`; sync and async. + +## Verification + +tests/test_pipeline, tests/test_core/{streams,nodes,operators,function_pod,side_effect_pod}, +golden suite on 3.12; async files on 3.11; ruff, basedpyright, changelog/design-issue checks. + +## Progress notes + +- Merge commit made locally (`Merge origin/main (ITL-912, #375) into ITL-1095`). +- Coordinator approved the xfail test change (1) and Option A for (2). +- (2) Pre-existing on main (checked against an export of origin/main): a Join node's zero-row + table has the populated table's fields in another order (data before system tags). The new + test compares that one case by field name, with a strict xfail of the exact order + (`test_a_join_with_no_input_rows_keeps_its_column_order`, "ITL-1131" until the + coordinator's issue id arrives). Reproduction: scratchpad `order.py` (filter -> join, sync and async). diff --git a/tests/test_pipeline/test_current_run_view_zero_row_types.py b/tests/test_pipeline/test_current_run_view_zero_row_types.py new file mode 100644 index 000000000..049f52b81 --- /dev/null +++ b/tests/test_pipeline/test_current_run_view_zero_row_types.py @@ -0,0 +1,198 @@ +"""A job node's current-run view has one Arrow schema in every state (ITL-1095, ITL-912). + +ITL-1095: a job node shows its current run, and reads zero rows before a run, after +``clear_cache()``, after a run that raised and while stale. ITL-912: a table with no rows +has exactly the Arrow schema of the node's rows (types, nullability, system tags, source +info, column order), derived from the input schemas. Together: ``as_table(all_info=True)`` +of a function node and of operator nodes (``group_by``, ``join``) has the same schema in +each state, over inputs whose types are not the defaults (an ``int8`` non-null tag, an +``int32`` data column). +""" + +from __future__ import annotations + +from typing import Any + +import pyarrow as pa +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.sources import ArrowTableSource +from orcapod.system_constants import constants + +ORCHESTRATORS = {"sync": SyncPipelineOrchestrator, "async": AsyncPipelineOrchestrator} +NODES = ("inc", "grouped", "joined") +FAIL = {"on": False} + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + """Return ``x + 1``; raise while ``FAIL["on"]`` is set.""" + if FAIL["on"]: + raise RuntimeError("inc failed on purpose") + return x + 1 + + +@pytest.fixture(autouse=True) +def _reset_failure() -> None: + FAIL["on"] = False + + +def _source(data: str, source_id: str) -> ArrowTableSource: + """Tags ``g``, ``id`` (``int8`` non-null) and one ``int32`` non-null data column.""" + schema = pa.schema( + [ + pa.field("g", pa.int8(), nullable=False), + pa.field("id", pa.int8(), nullable=False), + pa.field(data, pa.int32(), nullable=False), + ] + ) + table = pa.table( + {"g": [0, 1, 0, 1], "id": [0, 1, 2, 3], data: [10, 20, 30, 40]}, schema=schema + ) + return ArrowTableSource(table, tag_columns=["g", "id"], source_id=source_id) + + +def _job(db: InMemoryArrowDatabase | None = None) -> PipelineJob: + """``inc`` (function) -> ``grouped`` (GroupBy on ``g``); ``inc`` joined with ``b``.""" + job = PipelineJob(name="j", database=db or InMemoryArrowDatabase()) + with job: + rowwise: Any = inc.pod(_source("x", "a"), label="inc") + _ = rowwise.group_by("g", label="grouped") + _ = rowwise.join(_source("z", "b"), label="joined") + return job + + +def _assert_reads_with( + job: PipelineJob, expected: dict[str, pa.Schema], *, rows: bool +) -> None: + """Each node reads rows (or none) and has exactly the ``expected`` schema.""" + for label in NODES: + table = job.nodes[label].as_table(all_info=True) + assert (table.num_rows > 0) is rows, label + assert table.schema.equals(expected[label]), ( + label, + table.schema, + expected[label], + ) + + +def _assert_empty_with(job: PipelineJob, expected: dict[str, pa.Schema]) -> None: + _assert_reads_with(job, expected, rows=False) + + +def _run_schemas(orchestrator: str) -> dict[str, pa.Schema]: + """The schema of each node's populated table after a run of a fresh job.""" + job = _job() + _ = job.run(orchestrator=ORCHESTRATORS[orchestrator]()) + schemas: dict[str, pa.Schema] = {} + for label in NODES: + table = job.nodes[label].as_table(all_info=True) + assert table.num_rows > 0, label + schemas[label] = table.schema + return schemas + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +class TestOneSchemaInEveryState: + def test_the_populated_tables_keep_the_narrow_input_types( + self, orchestrator: str + ) -> None: + """The reference schemas carry the input's types and the system tags.""" + schemas = _run_schemas(orchestrator) + for label in ("inc", "joined"): + assert schemas[label].field("id").type == pa.int8(), label + assert not schemas[label].field("id").nullable, label + assert schemas["grouped"].field("g").type == pa.int8() + assert schemas["joined"].field("z").type == pa.int32() + for label in NODES: + assert any( + n.startswith(constants.SYSTEM_TAG_PREFIX) for n in schemas[label].names + ), label + + def test_before_a_run(self, orchestrator: str) -> None: + _assert_empty_with(_job(), _run_schemas(orchestrator)) + + def test_after_a_run_and_after_clear_cache(self, orchestrator: str) -> None: + expected = _run_schemas(orchestrator) + job = _job() + _ = job.run(orchestrator=ORCHESTRATORS[orchestrator]()) + _assert_reads_with(job, expected, rows=True) + for label in NODES: + job.nodes[label].clear_cache() + _assert_empty_with(job, expected) + + def test_after_a_failed_run(self, orchestrator: str) -> None: + expected = _run_schemas(orchestrator) + job = _job() + FAIL["on"] = True + with pytest.raises(RuntimeError, match="inc failed on purpose"): + _ = job.run( + orchestrator=ORCHESTRATORS[orchestrator](error_policy="fail_fast") + ) + _assert_empty_with(job, expected) + + def test_while_stale(self, orchestrator: str) -> None: + expected = _run_schemas(orchestrator) + job = _job() + _ = job.run(orchestrator=ORCHESTRATORS[orchestrator]()) + # A new source object of the same schema makes every node below it stale. + slot = next(name for name, src in job.sources.items() if "x" in src.keys()[1]) + job.bind(sources={slot: _source("x", "a")}) + for label in NODES: + assert job.nodes[label].is_stale, label + _assert_empty_with(job, expected) + + def test_after_a_load_only_run(self, orchestrator: str) -> None: + expected = _run_schemas(orchestrator) + db = InMemoryArrowDatabase() + _ = _job(db).run(orchestrator=ORCHESTRATORS[orchestrator]()) + job = _job(db) + FAIL["on"] = True # a load-only run never calls the function + _ = job.run(orchestrator=ORCHESTRATORS[orchestrator](), load_only=True) + _assert_reads_with(job, expected, rows=True) + + def test_after_a_load_only_run_with_nothing_stored(self, orchestrator: str) -> None: + """A load-only run over an empty store fills every node with zero rows. + + The join's zero-row result has its populated table's fields in another order + (ITL-1131), so the fields are compared by name here; the exact + schema is ``test_a_join_with_no_input_rows_keeps_its_column_order``. + """ + expected = _run_schemas(orchestrator) + job = _load_only_over_an_empty_store(orchestrator) + for label in NODES: + table = job.nodes[label].as_table(all_info=True) + assert table.num_rows == 0, label + assert _fields_by_name(table.schema) == _fields_by_name(expected[label]) + if label != "joined": + assert table.schema.equals(expected[label]), label + + @pytest.mark.xfail( + strict=True, + raises=AssertionError, + reason=( + "ITL-1131: a join's zero-row result lists its data columns before " + "its system tags; its populated table lists them after" + ), + ) + def test_a_join_with_no_input_rows_keeps_its_column_order( + self, orchestrator: str + ) -> None: + expected = _run_schemas(orchestrator)["joined"] + job = _load_only_over_an_empty_store(orchestrator) + assert job.nodes["joined"].as_table(all_info=True).schema.equals(expected) + + +def _load_only_over_an_empty_store(orchestrator: str) -> PipelineJob: + """A job after a load-only run over an empty store: every node holds zero rows.""" + job = _job() + FAIL["on"] = True # a load-only run never calls the function + _ = job.run(orchestrator=ORCHESTRATORS[orchestrator](), load_only=True) + return job + + +def _fields_by_name(schema: pa.Schema) -> dict[str, "pa.Field[Any]"]: + return {field.name: field for field in schema} From 8c44199bc68ae2e3c1a1c75c2c2fd9e4f179337a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 12:05:32 -0800 Subject: [PATCH 57/63] test(pipeline): docstrings for the current-run view schema tests (ITL-1095) CodeRabbit review of 7de3512b..410420e7. Co-Authored-By: Claude Opus 5.5 --- .../test_current_run_view_zero_row_types.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_pipeline/test_current_run_view_zero_row_types.py b/tests/test_pipeline/test_current_run_view_zero_row_types.py index 049f52b81..90ef12dbf 100644 --- a/tests/test_pipeline/test_current_run_view_zero_row_types.py +++ b/tests/test_pipeline/test_current_run_view_zero_row_types.py @@ -37,6 +37,7 @@ def inc(x: int) -> int: @pytest.fixture(autouse=True) def _reset_failure() -> None: + """Let ``inc`` succeed at the start of each test.""" FAIL["on"] = False @@ -80,6 +81,7 @@ def _assert_reads_with( def _assert_empty_with(job: PipelineJob, expected: dict[str, pa.Schema]) -> None: + """Each node reads zero rows and has exactly the ``expected`` schema.""" _assert_reads_with(job, expected, rows=False) @@ -97,6 +99,8 @@ def _run_schemas(orchestrator: str) -> dict[str, pa.Schema]: @pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) class TestOneSchemaInEveryState: + """``as_table(all_info=True)`` of every node keeps its run's schema in each state.""" + def test_the_populated_tables_keep_the_narrow_input_types( self, orchestrator: str ) -> None: @@ -113,9 +117,11 @@ def test_the_populated_tables_keep_the_narrow_input_types( ), label def test_before_a_run(self, orchestrator: str) -> None: + """A fresh job: every node reads zero rows with its run's schema.""" _assert_empty_with(_job(), _run_schemas(orchestrator)) def test_after_a_run_and_after_clear_cache(self, orchestrator: str) -> None: + """A run fills each node; ``clear_cache()`` empties it with the same schema.""" expected = _run_schemas(orchestrator) job = _job() _ = job.run(orchestrator=ORCHESTRATORS[orchestrator]()) @@ -125,6 +131,7 @@ def test_after_a_run_and_after_clear_cache(self, orchestrator: str) -> None: _assert_empty_with(job, expected) def test_after_a_failed_run(self, orchestrator: str) -> None: + """A run that raised leaves every node empty, with its run's schema.""" expected = _run_schemas(orchestrator) job = _job() FAIL["on"] = True @@ -135,6 +142,7 @@ def test_after_a_failed_run(self, orchestrator: str) -> None: _assert_empty_with(job, expected) def test_while_stale(self, orchestrator: str) -> None: + """A rebound source makes every node stale and empty, with the same schema.""" expected = _run_schemas(orchestrator) job = _job() _ = job.run(orchestrator=ORCHESTRATORS[orchestrator]()) @@ -146,6 +154,7 @@ def test_while_stale(self, orchestrator: str) -> None: _assert_empty_with(job, expected) def test_after_a_load_only_run(self, orchestrator: str) -> None: + """A load-only run over a store filled by a run shows that run's schema.""" expected = _run_schemas(orchestrator) db = InMemoryArrowDatabase() _ = _job(db).run(orchestrator=ORCHESTRATORS[orchestrator]()) @@ -181,6 +190,7 @@ def test_after_a_load_only_run_with_nothing_stored(self, orchestrator: str) -> N def test_a_join_with_no_input_rows_keeps_its_column_order( self, orchestrator: str ) -> None: + """The join that received no rows has its populated schema, in order.""" expected = _run_schemas(orchestrator)["joined"] job = _load_only_over_an_empty_store(orchestrator) assert job.nodes["joined"].as_table(all_info=True).schema.equals(expected) @@ -195,4 +205,5 @@ def _load_only_over_an_empty_store(orchestrator: str) -> PipelineJob: def _fields_by_name(schema: pa.Schema) -> dict[str, "pa.Field[Any]"]: + """The fields of ``schema`` by name, so that their order is not compared.""" return {field.name: field for field in schema} From 8455c2f027a30fd836646439710c10f20a0e4633 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 12:32:00 -0800 Subject: [PATCH 58/63] docs(claude): a failed end-of-run commit of node.run() clears only the nodes that ran (ITL-1121) Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 2 +- CLAUDE.md | 2 +- .../2026-11-14-itl-1121-node-view-review-fixes.md | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.zed/rules b/.zed/rules index 9d77d5a14..0e07fe720 100644 --- a/.zed/rules +++ b/.zed/rules @@ -817,7 +817,7 @@ data. The rules: non-stale run (for a full run: not load-only, nor anything above it) runs nothing; otherwise its non-current dependencies run first, each once, on the sync orchestrator (pipeline/node_run.py), and the run commits as job.run() does. A failed end-of-run - commit clears every node of the run. A load-only run's rows have status + commit clears only the nodes that ran; nodes it held keep their earlier run. A load-only run's rows have status loaded_only. Only job nodes' run() takes an argument (load_only). A loaded job (PipelineJob.load) runs only with load_only=True. job.run()/run_async() use the same state rule and plan (node_run.plan_run): a current node is diff --git a/CLAUDE.md b/CLAUDE.md index fc7d28484..9db54ec98 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -859,7 +859,7 @@ data. The rules: non-stale run (for a full run: not load-only, nor anything above it) runs nothing; otherwise its non-current dependencies run first, each once, on the sync orchestrator (`pipeline/node_run.py`), and the run commits as `job.run()` does. A failed end-of-run - commit clears every node of the run. A load-only run's rows have status + commit clears only the nodes that ran; nodes it held keep their earlier run. A load-only run's rows have status `loaded_only`. Only job nodes' `run()` takes an argument (`load_only`). A loaded job (`PipelineJob.load`) runs only with `load_only=True`. `job.run()`/`run_async()` use the same state rule and plan (`node_run.plan_run`): a current node is diff --git a/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md b/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md index 43607b34a..e96dc2bcc 100644 --- a/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md +++ b/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md @@ -154,3 +154,17 @@ Reproduction scripts: `superpowers/scripts/itl_1121/c1..c6_*.py` (committed). computing outside stream (d06ffcf7, node_run.refuse_unloadable_inputs). main merged (#382, #383). LEFT: name the coordinator's follow-up issue in the reply to finding G (thread 4574191943); CI and a CodeRabbit review of the head; worker_done. + +## Merge of main after #380 merged (task_f017ca8a173c) + +#380 merged as e4b4319c (it had merged #375 ITL-912 typed zero-row tables and #387 ITL-1111 +schema memo). Steps: +- [ ] `git merge origin/main` (no rebase); resolve function_node.py, operator_node.py, + side_effects.py, node_run.py, job.py, CLAUDE.md, .zed/rules keeping every #388 fix and main's + #375 / #387 behaviour. +- [ ] Doc fix: CLAUDE.md / .zed/rules "A failed end-of-run commit clears every node of the run" + -> only the nodes that ran (`run_with_dependencies` -> `commit_run(stores, plan.to_run)`). +- [ ] Re-run every script in superpowers/scripts/itl_1121/: each shows the fixed behaviour. +- [ ] Tests: tests/test_pipeline, tests/test_core/{nodes,operators,function_pod,side_effect_pod}, + golden (3.12; async files also 3.11; one process). +- [ ] One push; CI green; one signed `@coderabbitai review`; plan to coordinator before code. From d35d88914e2bb0fde0ee846e4ee5a80897438d00 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 12:41:28 -0800 Subject: [PATCH 59/63] docs(plan): ITL-1121 merge of main after #380 merged Co-Authored-By: Claude Opus 5.5 --- .../2026-11-14-itl-1121-node-view-review-fixes.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md b/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md index e96dc2bcc..83ab536c1 100644 --- a/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md +++ b/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md @@ -159,12 +159,18 @@ Reproduction scripts: `superpowers/scripts/itl_1121/c1..c6_*.py` (committed). #380 merged as e4b4319c (it had merged #375 ITL-912 typed zero-row tables and #387 ITL-1111 schema memo). Steps: -- [ ] `git merge origin/main` (no rebase); resolve function_node.py, operator_node.py, +- [x] `git merge origin/main` (no rebase); resolve function_node.py, operator_node.py, side_effects.py, node_run.py, job.py, CLAUDE.md, .zed/rules keeping every #388 fix and main's #375 / #387 behaviour. -- [ ] Doc fix: CLAUDE.md / .zed/rules "A failed end-of-run commit clears every node of the run" +- [x] Doc fix: CLAUDE.md / .zed/rules "A failed end-of-run commit clears every node of the run" -> only the nodes that ran (`run_with_dependencies` -> `commit_run(stores, plan.to_run)`). -- [ ] Re-run every script in superpowers/scripts/itl_1121/: each shows the fixed behaviour. -- [ ] Tests: tests/test_pipeline, tests/test_core/{nodes,operators,function_pod,side_effect_pod}, +- [x] Re-run every script in superpowers/scripts/itl_1121/: each shows the fixed behaviour. +- [x] Tests: tests/test_pipeline, tests/test_core/{nodes,operators,function_pod,side_effect_pod}, golden (3.12; async files also 3.11; one process). - [ ] One push; CI green; one signed `@coderabbitai review`; plan to coordinator before code. +- 2026-11-14 (merge task): merged #380's final head 8c44199b first (1aa7ca2f, no conflicts: + the true base), then origin/main e4b4319c (6b5db050; main's tree equals 8c44199b, so every + conflict of the squash took this branch's side). Doc fix 8455c2f0. All itl_1121 scripts show + the fixed behaviour. Tests: pipeline/nodes/operators/function_pod/side_effect_pod/golden on + 3.12 6434 passed 11 skipped 9 xfailed; async files + review-fix tests on 3.11 651 passed. + LEFT: CI, one CodeRabbit review, plan to coordinator, worker_done. From 03a87178a7e93f6caada531fb7f495502318706b Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 13:09:33 -0800 Subject: [PATCH 60/63] docs(node_run): a job node behind an outside stream is read, not run (ITL-1121) The _streams_read_by docstring said such a node is run as a node of its own; node.run() reads the rows it holds. Adds the c11 reproduction of CodeRabbit's finding 4575098786 (#380 behaviour, out of scope here). Co-Authored-By: Claude Opus 5.5 --- src/orcapod/pipeline/node_run.py | 3 +- ...6-11-14-itl-1121-node-view-review-fixes.md | 1 + .../c11_job_node_behind_outside_stream.py | 54 +++++++++++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 superpowers/scripts/itl_1121/c11_job_node_behind_outside_stream.py diff --git a/src/orcapod/pipeline/node_run.py b/src/orcapod/pipeline/node_run.py index 3efb70f31..1e3205972 100644 --- a/src/orcapod/pipeline/node_run.py +++ b/src/orcapod/pipeline/node_run.py @@ -53,7 +53,8 @@ def _streams_read_by(node: Any) -> Iterator[Any]: The node's inputs, the stream bound to a source slot among them, and, through every stream outside a job, that stream's own inputs; a job node other than a source slot - is not entered (the run reads its held rows or runs it as a node of its own). + is not entered: the run reads the rows the node holds (none if it has not run in + this process). Args: node: A job node; for a source slot, its bound stream is what it reads. diff --git a/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md b/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md index 83ab536c1..276cfc2a2 100644 --- a/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md +++ b/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md @@ -174,3 +174,4 @@ schema memo). Steps: the fixed behaviour. Tests: pipeline/nodes/operators/function_pod/side_effect_pod/golden on 3.12 6434 passed 11 skipped 9 xfailed; async files + review-fix tests on 3.11 651 passed. LEFT: CI, one CodeRabbit review, plan to coordinator, worker_done. +- 2026-11-14: CodeRabbit 4575098786 (5dd8d6a3 round): #380 behaviour, also on main; declined with repro c11 (coordinator files it); _streams_read_by docstring corrected. CR review at 21:07Z rate limited; retries ~21:28Z, ~21:48Z. diff --git a/superpowers/scripts/itl_1121/c11_job_node_behind_outside_stream.py b/superpowers/scripts/itl_1121/c11_job_node_behind_outside_stream.py new file mode 100644 index 000000000..7f63d1938 --- /dev/null +++ b/superpowers/scripts/itl_1121/c11_job_node_behind_outside_stream.py @@ -0,0 +1,54 @@ +"""CodeRabbit 4575098786 (out of scope, #380 behaviour, also on main e4b4319c): node.run() +does not schedule a job node reached through a stream outside a job; the run reads 0 rows.""" + +import pyarrow as pa +from orcapod import function_pod +from orcapod.core.nodes.function_node import FunctionJobNode +from orcapod.core.nodes.operator_node import OperatorJobNode +from orcapod.core.operators import MapData +from orcapod.core.streams import ArrowTableStream +from orcapod.databases import InMemoryArrowDatabase + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + return x + 1 + + +@function_pod(output_keys="w") +def dbl(z: int) -> int: + return 2 * z + + +schema = pa.schema( + [ + pa.field("id", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] +) +base = ArrowTableStream( + pa.table({"id": [1, 2, 3], "x": [1, 2, 3]}, schema=schema), tag_columns=["id"] +) +for via in ["function", "operator"]: + first = FunctionJobNode(inc.pod, base, pipeline_database=InMemoryArrowDatabase()) + mid = MapData({"y": "z"})( + first + ) # an operator's stream outside a job over a job node + if via == "function": + second = FunctionJobNode( + dbl.pod, mid, pipeline_database=InMemoryArrowDatabase() + ) + else: + second = OperatorJobNode( + operator=MapData({"z": "q"}), + input_streams=(mid,), + pipeline_database=InMemoryArrowDatabase(), + ) + print(type(mid).__name__, "first ran?", first.holds_current_run(load_only=False)) + try: + view = second.run() + print( + f"{via}: second.run() rows={view.as_table().num_rows}; first holds a run: {first.holds_current_run(load_only=False)}; summary: {second.last_run}" + ) + except Exception as exc: + print(f"{via}: raised {type(exc).__name__}: {exc}") From 43bf5fdc4bec348ac483e65e89eda6d53c67d0d6 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 13:36:37 -0800 Subject: [PATCH 61/63] test(scripts): type the c11 reproduction's pods for basedpyright (ITL-1121) Co-Authored-By: Claude Opus 5.5 --- .../itl_1121/c11_job_node_behind_outside_stream.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/superpowers/scripts/itl_1121/c11_job_node_behind_outside_stream.py b/superpowers/scripts/itl_1121/c11_job_node_behind_outside_stream.py index 7f63d1938..f7e78116e 100644 --- a/superpowers/scripts/itl_1121/c11_job_node_behind_outside_stream.py +++ b/superpowers/scripts/itl_1121/c11_job_node_behind_outside_stream.py @@ -1,6 +1,8 @@ """CodeRabbit 4575098786 (out of scope, #380 behaviour, also on main e4b4319c): node.run() does not schedule a job node reached through a stream outside a job; the run reads 0 rows.""" +from typing import Any + import pyarrow as pa from orcapod import function_pod from orcapod.core.nodes.function_node import FunctionJobNode @@ -20,6 +22,9 @@ def dbl(z: int) -> int: return 2 * z +inc_pod: Any = inc.pod +dbl_pod: Any = dbl.pod + schema = pa.schema( [ pa.field("id", pa.int64(), nullable=False), @@ -30,13 +35,13 @@ def dbl(z: int) -> int: pa.table({"id": [1, 2, 3], "x": [1, 2, 3]}, schema=schema), tag_columns=["id"] ) for via in ["function", "operator"]: - first = FunctionJobNode(inc.pod, base, pipeline_database=InMemoryArrowDatabase()) + first = FunctionJobNode(inc_pod, base, pipeline_database=InMemoryArrowDatabase()) mid = MapData({"y": "z"})( first ) # an operator's stream outside a job over a job node if via == "function": second = FunctionJobNode( - dbl.pod, mid, pipeline_database=InMemoryArrowDatabase() + dbl_pod, mid, pipeline_database=InMemoryArrowDatabase() ) else: second = OperatorJobNode( From 9fc663d743cda132ddfcab68f69b8fdb24ba46d7 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 13:55:07 -0800 Subject: [PATCH 62/63] docs(plan): ITL-1121 merges of #390 and #389 Co-Authored-By: Claude Opus 5.5 --- superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md b/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md index 276cfc2a2..2848b9a56 100644 --- a/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md +++ b/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md @@ -175,3 +175,4 @@ schema memo). Steps: 3.12 6434 passed 11 skipped 9 xfailed; async files + review-fix tests on 3.11 651 passed. LEFT: CI, one CodeRabbit review, plan to coordinator, worker_done. - 2026-11-14: CodeRabbit 4575098786 (5dd8d6a3 round): #380 behaviour, also on main; declined with repro c11 (coordinator files it); _streams_read_by docstring corrected. CR review at 21:07Z rate limited; retries ~21:28Z, ~21:48Z. +- 2026-11-14: merged #390 (653c138e; one conflict in OperatorJobNode.run, kept _compute_output + _run_load_only) and #389 (810d1b03, clean). Scripts unchanged. 3.12 targets 9348 passed; async 3.11 651 passed. CR retry 1 at 21:38Z rate limited; last retry ~21:58Z. From 83350de6270fc73b91e93d420ce1251855879ad6 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 14:28:47 -0800 Subject: [PATCH 63/63] fix(node_run): the load-only refusal reads a held table and refuses a SideEffectNode (ITL-1121) CodeRabbit round 2 on #388. An ArrowTableStream reads only its table, so the walk no longer enters its upstreams (a load-only run over one built with a function pod's stream among its upstreams was refused). A SideEffectNode outside a job delivers when it is read, so a load-only run refuses it as it refuses a side-effect pod's stream (it delivered as a node's input and bound to a source slot). Reproduction: superpowers/scripts/itl_1121/c12_*. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1121.md | 2 +- src/orcapod/pipeline/node_run.py | 17 +-- ...6-11-14-itl-1121-node-view-review-fixes.md | 1 + .../itl_1121/c12_cr388_round2_traversal.py | 101 ++++++++++++++++++ .../test_node_view_review_fixes.py | 88 +++++++++++++++ 5 files changed, 202 insertions(+), 7 deletions(-) create mode 100644 superpowers/scripts/itl_1121/c12_cr388_round2_traversal.py diff --git a/changelog.d/ITL-1121.md b/changelog.d/ITL-1121.md index c4f5ae9d3..b0ff92bde 100644 --- a/changelog.d/ITL-1121.md +++ b/changelog.d/ITL-1121.md @@ -1,6 +1,6 @@ ### Changed -- A load-only run (`job.run(load_only=True)`, `run_async(load_only=True)`, `node.run(load_only=True)`, a source slot's `run(load_only=True)`) reads only job nodes, sources and streams that neither compute nor deliver (developer, November 14, 2026; ITL-1121). A pod's stream built outside the job is refused with a `ValueError` before anything is read: a function pod's stream (a cached pod's included), which computes when it is read, and a side-effect pod's stream, which delivers its side effects (before, the run read it and delivered every row). This applies whether a job node reads the stream, a source slot is bound to it or an operator's stream outside the job reads it. Build the pod inside the job, or run without `load_only`; a run without `load_only` is unchanged. ITL-1130 re-evaluates the choice. +- A load-only run (`job.run(load_only=True)`, `run_async(load_only=True)`, `node.run(load_only=True)`, a source slot's `run(load_only=True)`) reads only job nodes, sources and streams that neither compute nor deliver (developer, November 14, 2026; ITL-1121). A pod's stream built outside the job is refused with a `ValueError` before anything is read: a function pod's stream (a cached pod's included), which computes when it is read, and a side-effect pod's stream or a `SideEffectNode` outside a job, which deliver their side effects (before, the run read them and delivered every row). An `ArrowTableStream` holds its rows and is read whatever its `upstreams` name. This applies whether a job node reads the stream, a source slot is bound to it or an operator's stream outside the job reads it. Build the pod inside the job, or run without `load_only`; a run without `load_only` is unchanged. ITL-1130 re-evaluates the choice. ### Fixed diff --git a/src/orcapod/pipeline/node_run.py b/src/orcapod/pipeline/node_run.py index 1e3205972..2a0f18119 100644 --- a/src/orcapod/pipeline/node_run.py +++ b/src/orcapod/pipeline/node_run.py @@ -52,14 +52,16 @@ def _streams_read_by(node: Any) -> Iterator[Any]: """Yield the streams outside a job that a run of ``node`` reads. The node's inputs, the stream bound to a source slot among them, and, through every - stream outside a job, that stream's own inputs; a job node other than a source slot + stream outside a job, that stream's own inputs. A job node other than a source slot is not entered: the run reads the rows the node holds (none if it has not run in - this process). + this process). Nor is an ``ArrowTableStream``: reading it reads only the table it + holds, whatever its ``upstreams`` name (ITL-1121). Args: node: A job node; for a source slot, its bound stream is what it reads. """ from orcapod.core.nodes.source_node import SourceJobNode + from orcapod.core.streams.arrow_table_stream import ArrowTableStream pending: list[Any] = ( [node.bound_source] if isinstance(node, SourceJobNode) else list(node.upstreams) @@ -76,7 +78,8 @@ def _streams_read_by(node: Any) -> Iterator[Any]: if _is_job_node(stream): continue yield stream - pending.extend(getattr(stream, "upstreams", ())) + if not isinstance(stream, ArrowTableStream): + pending.extend(getattr(stream, "upstreams", ())) def refuse_unloadable_inputs(node: Any) -> None: @@ -87,7 +90,8 @@ def refuse_unloadable_inputs(node: Any) -> None: job nodes, sources and streams that neither compute nor deliver (developer, November 14, 2026: "Refuse is consistent"; ITL-1121). Outside a job, a function pod's stream (``FunctionPodStream``, a cached pod's included) computes its rows when - read, and a side-effect pod's stream (``SideEffectPodStream``) delivers its side + read, and a side-effect pod's stream (``SideEffectPodStream``) or node + (``SideEffectNode``, the blueprint a ``Pipeline`` records) delivers its side effects. Such a stream is refused before anything is read, whether it is an input of ``node``, bound to a source slot, or read through another stream outside a job (an operator's stream over it). @@ -99,10 +103,11 @@ def refuse_unloadable_inputs(node: Any) -> None: ValueError: If a run of ``node`` would read such a stream. """ from orcapod.core.function_pod import FunctionPodStream - from orcapod.side_effects import SideEffectPodStream + from orcapod.side_effects import SideEffectNode, SideEffectPodStream for upstream in _streams_read_by(node): - if isinstance(upstream, SideEffectPodStream): + # A SideEffectJobNode is a job node: _streams_read_by never yields one. + if isinstance(upstream, (SideEffectPodStream, SideEffectNode)): what = ( "a side-effect pod's stream outside a job, which delivers its side " "effects when it is read, and a load-only run delivers nothing" diff --git a/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md b/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md index 2848b9a56..08b6e892d 100644 --- a/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md +++ b/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md @@ -176,3 +176,4 @@ schema memo). Steps: LEFT: CI, one CodeRabbit review, plan to coordinator, worker_done. - 2026-11-14: CodeRabbit 4575098786 (5dd8d6a3 round): #380 behaviour, also on main; declined with repro c11 (coordinator files it); _streams_read_by docstring corrected. CR review at 21:07Z rate limited; retries ~21:28Z, ~21:48Z. - 2026-11-14: merged #390 (653c138e; one conflict in OperatorJobNode.run, kept _compute_output + _run_load_only) and #389 (810d1b03, clean). Scripts unchanged. 3.12 targets 9348 passed; async 3.11 651 passed. CR retry 1 at 21:38Z rate limited; last retry ~21:58Z. +- 2026-11-14: CodeRabbit round 2 (review 5907415238 of 9fc663d7): A 4575874462 (ArrowTableStream upstreams over-refused) and B 4575874471 (blueprint SideEffectNode delivered in a load-only run) reproduced (c12), approved, fixed in one commit with 13 tests. 3.12 targets 9438 passed; 3.11 async 664 passed. diff --git a/superpowers/scripts/itl_1121/c12_cr388_round2_traversal.py b/superpowers/scripts/itl_1121/c12_cr388_round2_traversal.py new file mode 100644 index 000000000..a24dace2a --- /dev/null +++ b/superpowers/scripts/itl_1121/c12_cr388_round2_traversal.py @@ -0,0 +1,101 @@ +"""PR #388 CodeRabbit round 2 (review of 9fc663d7): the load-only input traversal. + +A (4575874462): an ArrowTableStream holds its table and never reads its upstreams, yet a + load-only run over one built with upstreams=(a function pod's stream,) is refused. +B (4575874471): a blueprint SideEffectNode outside a job delivers when it is read; a + load-only run over it (a node's input, or bound to a source slot) is not refused. +""" + +from typing import Any + +import pyarrow as pa + +from orcapod import PipelineJob, function_pod +from orcapod.core.nodes.function_node import FunctionJobNode +from orcapod.core.streams import ArrowTableStream +from orcapod.databases import InMemoryArrowDatabase +from orcapod.side_effects import SideEffectNode, tap_pod +from orcapod.sources import DictSource + +UP: list[int] = [] +CALLS: list[int] = [] + + +def deliver_up(x: int, ctx: Any) -> None: + UP.append(x) + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + CALLS.append(x) + return x + 1 + + +@function_pod(output_keys="w") +def twice(x: int) -> int: + return 2 * x + + +inc_pod: Any = inc.pod +twice_pod: Any = twice.pod +schema = pa.schema( + [ + pa.field("k", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] +) + + +def base() -> Any: + return ArrowTableStream( + pa.table({"k": [1, 2], "x": [1, 2]}, schema=schema), tag_columns=["k"] + ) + + +def blueprint_tap() -> Any: + tap: Any = tap_pod(deliver_up) + return SideEffectNode(tap, base()) + + +# A +held = ArrowTableStream( + pa.table({"k": [1, 2], "x": [1, 2]}, schema=schema), + tag_columns=["k"], + upstreams=(inc_pod(base()),), +) +node = FunctionJobNode(twice_pod, held, pipeline_database=InMemoryArrowDatabase()) +try: + view = node.run(load_only=True) + print( + f"A: run(load_only=True) accepted: status {view.status}; function calls {len(CALLS)}" + ) +except ValueError as exc: + print(f"A: refused: {exc}; function calls {len(CALLS)}") + +# B, as a node's input +UP.clear() +node = FunctionJobNode( + twice_pod, blueprint_tap(), pipeline_database=InMemoryArrowDatabase() +) +try: + node.run(load_only=True) + print(f"B input: run(load_only=True) ran: delivered {len(UP)}") +except ValueError as exc: + print(f"B input: refused: {exc}; delivered {len(UP)}") + +# B, bound to a source slot +db = InMemoryArrowDatabase() +job = PipelineJob(name="j", database=db) +with job: + src = DictSource( + [{"k": 1, "x": 1}, {"k": 2, "x": 2}], tag_columns=["k"], source_id="s" + ) + twice.pod(src, label="tw") +job.run() +UP.clear() +try: + job.bind(sources={"s": blueprint_tap()}) + job.run(load_only=True) + print(f"B slot: job.run(load_only=True) ran: delivered {len(UP)}") +except ValueError as exc: + print(f"B slot: refused: {exc}; delivered {len(UP)}") diff --git a/tests/test_pipeline/test_node_view_review_fixes.py b/tests/test_pipeline/test_node_view_review_fixes.py index 631c49a9e..b3d90e9f9 100644 --- a/tests/test_pipeline/test_node_view_review_fixes.py +++ b/tests/test_pipeline/test_node_view_review_fixes.py @@ -593,3 +593,91 @@ def test_a_full_run_over_an_input_that_computes_reads_it(kind: str) -> None: node = _node_over(kind, _computing_stream("pod")) assert node.run().as_table().num_rows == 2 assert len(_UPSTREAM_CALLS) == 2 + + +# --------------------------------------------------------------------------- +# PR #388 review round 2: what the load-only refusal walks through +# --------------------------------------------------------------------------- + + +def _held_table_over_a_computing_stream() -> Any: + """An ``ArrowTableStream`` holding its rows, whose ``upstreams`` name a function pod's + stream outside a job; reading it reads only the held table.""" + import pyarrow as pa + + from orcapod.core.streams import ArrowTableStream + + schema = pa.schema( + [ + pa.field("id", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] + ) + return ArrowTableStream( + pa.table({"id": [1, 2], "x": [1, 2]}, schema=schema), + tag_columns=["id"], + upstreams=(_computing_stream("pod"),), + ) + + +@pytest.mark.parametrize("kind", ["side_effect", "function", "operator"]) +def test_a_load_only_run_reads_a_table_stream_whatever_its_upstreams( + kind: str, +) -> None: + """An ``ArrowTableStream`` neither computes nor delivers: not refused.""" + _UPSTREAM_CALLS.clear() + node = _node_over(kind, _held_table_over_a_computing_stream()) + node.run(load_only=True) + assert node.holds_current_run(load_only=True) + assert _UPSTREAM_CALLS == [] + + +def _blueprint_delivering_node() -> Any: + """A ``SideEffectNode`` outside a job: reading it delivers each row.""" + from orcapod.side_effects import SideEffectNode + + return SideEffectNode(_tap(_deliver_upstream), _base_stream()) + + +@pytest.mark.parametrize("kind", ["side_effect", "function", "operator"]) +def test_a_load_only_run_refuses_a_side_effect_node_outside_a_job(kind: str) -> None: + """Refused before the input is read: nothing is delivered.""" + _UPSTREAM_DELIVERED.clear() + node = _node_over(kind, _blueprint_delivering_node()) + with pytest.raises(ValueError, match="delivers its side effects when it is read"): + node.run(load_only=True) + assert _UPSTREAM_DELIVERED == [] + assert not node.holds_current_run(load_only=True) + + +@pytest.mark.parametrize("how", ["sync", "async", "node.run", "source.run"]) +def test_a_load_only_run_refuses_a_source_slot_bound_to_a_side_effect_node( + how: str, +) -> None: + """A source slot bound to a ``SideEffectNode`` outside a job: refused.""" + _UPSTREAM_DELIVERED.clear() + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + source = DictSource( + [{"id": 1, "x": 1}, {"id": 2, "x": 2}], tag_columns=["id"], source_id="s" + ) + _twice_pod(source, label="tw") + job.run() + job.bind(sources={"s": _blueprint_delivering_node()}) + with pytest.raises(ValueError, match="delivers its side effects when it is read"): + if how == "node.run": + job.nodes["tw"].run(load_only=True) + elif how == "source.run": + job.nodes["tw"].upstreams[0].run(load_only=True) + else: + job.run(orchestrator=ORCHESTRATORS[how](), load_only=True) + assert _UPSTREAM_DELIVERED == [] + + +@pytest.mark.parametrize("kind", ["side_effect", "function", "operator"]) +def test_a_full_run_over_a_side_effect_node_outside_a_job_reads_it(kind: str) -> None: + """Unchanged: a run without ``load_only`` reads it, which delivers.""" + _UPSTREAM_DELIVERED.clear() + node = _node_over(kind, _blueprint_delivering_node()) + assert node.run().as_table().num_rows == 2 + assert len(_UPSTREAM_DELIVERED) == 2