From 3c8a69cbe8e158b1a4c12195b28b1c3ce78088e4 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 20:38:56 -0800 Subject: [PATCH 01/48] test(function_node): failing tests for the rows a job node returns after a run (ITL-1093) Co-Authored-By: Claude Opus 5.5 --- ...-11-13-itl-1093-job-node-rows-after-run.md | 87 +++++ .../test_job_node_rows_after_run.py | 300 ++++++++++++++++++ 2 files changed, 387 insertions(+) create mode 100644 superpowers/plans/2026-11-13-itl-1093-job-node-rows-after-run.md create mode 100644 tests/test_pipeline/test_job_node_rows_after_run.py diff --git a/superpowers/plans/2026-11-13-itl-1093-job-node-rows-after-run.md b/superpowers/plans/2026-11-13-itl-1093-job-node-rows-after-run.md new file mode 100644 index 000000000..bc3428932 --- /dev/null +++ b/superpowers/plans/2026-11-13-itl-1093-job-node-rows-after-run.md @@ -0,0 +1,87 @@ +# ITL-1093: a function job node returns the rows of its run's inputs — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** After a run on either orchestrator, a `FunctionJobNode` returns exactly the rows of +that run's inputs (`as_table`, `iter_data`, `head`, `run()`, a consumer built over it), row for +row equal between sync and async. Nothing stored changes. + +**Architecture:** `FunctionJobNode._cached_output_datas` stays a superset cache keyed by base +entry id. A new `_run_entry_ids: list[bytes] | None` records the base entry ids of the last +completed run's inputs in input order (`None` = no completed run in this process). One helper +`_held_pairs()` is the only reader of the held rows: with run ids it yields one row per run +input (not deduplicated, so held rows equal what downstream received), else every held value +(unchanged no-run behaviour). A run that raises defines no rows: the node clears its in-memory +rows and run marker (`clear_cache()`) and reads as a no-run node. + +**Tech Stack:** Python 3.11+, PyArrow, pytest, `uv run`. + +Coordinator decisions (November 13, 2026): proposal approved; no-run read unchanged (option A, +documented; B/C reported to the developer); duplicates not deduplicated; a raised run → no-run +read (option i); REPLAY union and async `fail_fast` (ITL-879) reported only. + +--- + +## Cause (measured on main b4fde330) + +- `async_execute` loads `self._load_cached_entries()` with no ids (every pipeline record under + the node's content hash) into `_cached_output_datas`; every read yields every dict value. +- Sync `execute` loads only this run's ids (`get_cached_results(base_entry_ids)`), so it looked + right only because a new job builds fresh node objects. Re-running the same job object with a + rebound source accumulates on sync too (3/6/6/9). + +## Files + +- Modify: `src/orcapod/core/nodes/function_node.py` (FunctionJobNode: `__init__`, `clear_cache`, + `execute`, `async_execute`, `iter_data`, `head`, new `_held_pairs`) +- Create: `tests/test_pipeline/test_job_node_rows_after_run.py` +- Create: `changelog.d/ITL-1093.md`, `design_issues/ITL-1093-JN1.md` +- Modify: `CLAUDE.md`, `.zed/rules` (Phase 1 sentence); `docs/concepts/` sentence if one exists. + +## Tasks + +### Task 1: Failing tests + +- [ ] Write `tests/test_pipeline/test_job_node_rows_after_run.py`: pipeline `rowwise` (inc over + DictSource source_id "s"), `downstream` (dbl over rowwise), `grouped` (total over + `src.group_by("g")`). Sequence 3 / 6 / 3 again / 3 different rows. + - fresh job per run, sync and async: each node's `as_table`, `iter_data`, `head(100)`, + `run()` and a consumer `dbl.pod(job.nodes["rowwise"])` return the rows of the run's inputs; + sync == async as sets of rows with system tags; the 6-row run reports 3 reused. + - same job object, `job.bind(sources=...)` per run, sync and async: same expectations. + - filtered inputs (function returns None) are in no read. + - a raised run (sync fail_fast ValueError; sync and async StoreWriteError raised part-way) + reads as a no-run node: equal to a fresh compiled job's read over the same store. +- [ ] Run (Python 3.12 and 3.11): `uv run pytest tests/test_pipeline/test_job_node_rows_after_run.py` + — expect async and same-job cases to FAIL. +- [ ] WIP commit. + +### Task 2: Fix + +- [ ] `_run_entry_ids` class default `None`, set in `__init__`; `clear_cache()` resets it. +- [ ] `_held_pairs()` helper; `iter_data` (both branches) and `head`'s held branch read through it; + `head` takes the held branch when `_run_entry_ids is not None` even with no held rows. +- [ ] `execute`: completed flag; at success set `_run_entry_ids = base_entry_ids` and drop the + cached table/hash column; on any raise after processing began, `_commit_after_failure` + (unchanged) then `clear_cache()`. +- [ ] `async_execute` DB path: `route_inputs` appends each input's base entry id to a local + list; at success set `_run_entry_ids`; a raise (Exception or cancellation) → `clear_cache()`. +- [ ] Run new tests (3.12 + 3.11) and targeted node/pipeline tests; commit. + +### Task 3: Docs and records + +- [ ] Docstrings of `iter_data`, `as_table`, `head`, `run`, `execute`, `async_execute` state the rule. +- [ ] `changelog.d/ITL-1093.md` under `### Fixed`; check with + `uv run python scripts/assemble_changelog.py --check`. +- [ ] `design_issues/ITL-1093-JN1.md` (resolved, Fix note); `uv run python scripts/design_issues.py --check`. +- [ ] CLAUDE.md + `.zed/rules` sentence → "After a run, a `FunctionJobNode` returns exactly the rows + of that run's inputs, sync or async (a run that raised defines no rows). Read without a + completed run in this process (fresh or loaded job), it returns every record stored under its + content hash for the current data context. A run skips inputs whose entry is already stored." +- [ ] docs/concepts sentence if one states what a job node returns after a run. +- [ ] ruff, ruff format, basedpyright (exit code), golden suite, targeted suites; commit; push; draft PR. + +## Notes for a successor + +- Probes: scratchpad (not durable); measured tables are in the coordinator ask and the PR body. +- PR #359 (ITL-1063) edits the same `async_execute`/`execute` hunks; semantics compatible. diff --git a/tests/test_pipeline/test_job_node_rows_after_run.py b/tests/test_pipeline/test_job_node_rows_after_run.py new file mode 100644 index 000000000..7f2ac88c8 --- /dev/null +++ b/tests/test_pipeline/test_job_node_rows_after_run.py @@ -0,0 +1,300 @@ +"""After a run, a function job node returns exactly the rows of that run's inputs (ITL-1093). + +One store, a source of one ``source_id`` holding 3 rows, then 6, then the same 3 again, then +3 different rows. After each run, on either orchestrator, every function job node returns +the rows of that run's inputs through ``as_table``, ``iter_data``, ``head`` and ``run()``, +and so does a consumer built over the node afterwards; the two orchestrators agree row for +row. Stored results of other inputs stay stored and reusable. A run that raised defines no +rows: the node then reads as a node with no run in this process (every row stored under +its content hash). +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.errors import StoreWriteError +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.sources import DictSource + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + """Return ``x + 1``.""" + return x + 1 + + +@function_pod(output_keys="z") +def dbl(y: int) -> int: + """Return ``2 * y``.""" + return 2 * y + + +@function_pod(output_keys="total") +def total(x: list[int]) -> int: + """Return the sum of a group's values.""" + return sum(x) + + +@function_pod(output_keys="y") +def null_on_10(x: int) -> int | None: + """Return ``x + 1``, or a null value for ``x == 10``.""" + return None if x == 10 else x + 1 + + +@function_pod(output_keys="y") +def fails_on_20(x: int) -> int: + """Return ``x + 1``; raise for ``x == 20``.""" + if x == 20: + raise ValueError("x is 20") + return x + 1 + + +@function_pod(output_keys="y") +def store_fails_on_20(x: int) -> int: + """Return ``x + 1``; for ``x == 20`` raise the error a failed store write raises. + + A ``StoreError`` fails the run whatever the error policy, on both orchestrators, as a + failed write of the node's results does. + """ + if x == 20: + raise StoreWriteError("simulated failed store write") + return x + 1 + + +ROWS = [{"g": i % 2, "k": i, "x": 10 * i} for i in range(6)] +# 3 rows, 6 rows (3 more), the same 3 again (fewer), 3 different rows. The source's record +# ids are positional, so the last run's rows are new inputs whose results are stored. +SEQUENCE = [ROWS[:3], ROWS, ROWS[:3], ROWS[3:]] +ORCHESTRATORS = ["sync", "async"] + + +def _source(data: list[dict[str, int]]) -> DictSource: + return DictSource(data, tag_columns=["g", "k"], source_id="s") + + +def _build(data: list[dict[str, int]], db: Any) -> PipelineJob: + job = PipelineJob(name="itl1093", database=db) + with job: + rowwise = inc.pod(_source(data), label="rowwise") + dbl.pod(rowwise, label="downstream") + total.pod(_source(data).group_by("g"), label="grouped") + return job + + +def _orchestrator(kind: str, error_policy: str = "continue") -> Any: + if kind == "sync": + return SyncPipelineOrchestrator(error_policy=error_policy) # pyright: ignore[reportArgumentType] + return AsyncPipelineOrchestrator(error_policy=error_policy) # pyright: ignore[reportArgumentType] + + +def _canonical(rows: list[dict[str, Any]]) -> list[tuple[tuple[str, str], ...]]: + """Rows as a sorted list, each a sorted tuple of (column, repr(value)).""" + return sorted(tuple(sorted((k, repr(v)) for k, v in row.items())) for row in rows) + + +ALL = {"system_tags": True} + + +def _reads(node: Any) -> dict[str, list[tuple[tuple[str, str], ...]]]: + """Every public read of ``node``, rows with system tags, canonical.""" + iterated = [ + {**tag.as_dict(all_info=True), **data.as_dict()} for tag, data in node.iter_data() + ] + return { + "as_table": _canonical(node.as_table(columns=ALL).to_pylist()), + "head": _canonical(node.head(100, columns=ALL).as_table().to_pylist()), + "iter_data": _canonical( + [ + {k: v for k, v in row.items() if not k.startswith("__")} + for row in iterated + ] + ), + } + + +def _values(node: Any, key: Callable[[dict[str, Any]], Any]) -> list[Any]: + return sorted(key(row) for row in node.as_table().to_pylist()) + + +def _expected_rowwise(data: list[dict[str, int]]) -> list[tuple[int, int]]: + return sorted((row["k"], row["x"] + 1) for row in data) + + +def _expected_downstream(data: list[dict[str, int]]) -> list[tuple[int, int]]: + return sorted((row["k"], 2 * (row["x"] + 1)) for row in data) + + +def _expected_grouped(data: list[dict[str, int]]) -> list[tuple[int, int]]: + sums: dict[int, int] = {} + for row in data: + sums[row["g"]] = sums.get(row["g"], 0) + row["x"] + return sorted(sums.items()) + + +def _check_run(job: PipelineJob, data: list[dict[str, int]]) -> None: + """Assert every node of ``job`` returns the rows of ``data``, through every read.""" + nodes = job.nodes + assert _values(nodes["rowwise"], lambda r: (r["k"], r["y"])) == _expected_rowwise( + data + ) + assert _values(nodes["downstream"], lambda r: (r["k"], r["z"])) == ( + _expected_downstream(data) + ) + assert _values(nodes["grouped"], lambda r: (r["g"], r["total"])) == ( + _expected_grouped(data) + ) + for label in ("rowwise", "downstream", "grouped"): + reads = _reads(nodes[label]) + assert reads["head"] == reads["as_table"], label + assert len(reads["iter_data"]) == len(reads["as_table"]), label + # A consumer built over the node after the run receives the run's rows. + consumer = dbl.pod(nodes["rowwise"]) + assert sorted((r["k"], r["z"]) for r in consumer.as_table().to_pylist()) == ( + _expected_downstream(data) + ) + + +def _snapshot(job: PipelineJob) -> dict[str, Any]: + return {label: _reads(job.nodes[label]) for label in ("rowwise", "downstream", "grouped")} + + +class TestFreshJobPerRun: + """A new job over the same store for every run, as the issue's reproduction.""" + + @pytest.mark.parametrize("kind", ORCHESTRATORS) + def test_each_run_returns_its_inputs_rows(self, kind: str) -> None: + db = InMemoryArrowDatabase() + for index, data in enumerate(SEQUENCE): + job = _build(data, db) + summary = job.run(orchestrator=_orchestrator(kind)) + _check_run(job, data) + if index == 1: + # The 6-row run reuses the 3 stored results of the first run. + assert summary.nodes["rowwise"].reused == 3 + assert summary.nodes["rowwise"].computed == 3 + + def test_sync_and_async_agree_row_for_row(self) -> None: + snapshots: dict[str, list[dict[str, Any]]] = {} + for kind in ORCHESTRATORS: + db = InMemoryArrowDatabase() + snapshots[kind] = [] + for data in SEQUENCE: + job = _build(data, db) + job.run(orchestrator=_orchestrator(kind)) + snapshots[kind].append(_snapshot(job)) + assert snapshots["sync"] == snapshots["async"] + + @pytest.mark.parametrize("kind", ORCHESTRATORS) + def test_node_run_returns_its_inputs_rows(self, kind: str) -> None: + db = InMemoryArrowDatabase() + for data in SEQUENCE: + job = _build(data, db) + job.run(orchestrator=_orchestrator(kind)) + view = job.nodes["rowwise"].run() + assert sorted((r["k"], r["y"]) for r in view.as_table().to_pylist()) == ( + _expected_rowwise(data) + ) + + +class TestSameJobRebound: + """One job object, its source rebound before every run: the same node objects run again.""" + + @pytest.mark.parametrize("kind", ORCHESTRATORS) + def test_each_run_returns_its_inputs_rows(self, kind: str) -> None: + db = InMemoryArrowDatabase() + job = _build(SEQUENCE[0], db) + for data in SEQUENCE: + job.bind(sources={"s": _source(data)}) + job.run(orchestrator=_orchestrator(kind)) + _check_run(job, data) + + def test_sync_and_async_agree_row_for_row(self) -> None: + snapshots: dict[str, list[dict[str, Any]]] = {} + for kind in ORCHESTRATORS: + db = InMemoryArrowDatabase() + job = _build(SEQUENCE[0], db) + snapshots[kind] = [] + for data in SEQUENCE: + job.bind(sources={"s": _source(data)}) + job.run(orchestrator=_orchestrator(kind)) + snapshots[kind].append(_snapshot(job)) + assert snapshots["sync"] == snapshots["async"] + + +def _single(fn: Any, data: list[dict[str, int]], db: Any) -> PipelineJob: + job = PipelineJob(name="itl1093_single", database=db) + with job: + fn.pod(_source(data), label="node") + return job + + +class TestNullAndInactive: + """A null value is a row; an inactive function's inputs (no output) are rows of no read.""" + + @pytest.mark.parametrize("kind", ORCHESTRATORS) + def test_null_value_is_a_row(self, kind: str) -> None: + db = InMemoryArrowDatabase() + _single(null_on_10, ROWS, db).run(orchestrator=_orchestrator(kind)) + job = _single(null_on_10, ROWS[:3], db) + job.run(orchestrator=_orchestrator(kind)) + node = job.nodes["node"] + assert _values(node, lambda r: (r["k"], r["y"])) == [(0, 1), (1, None), (2, 21)] + reads = _reads(node) + assert reads["head"] == reads["as_table"] + assert len(reads["iter_data"]) == 3 + + @pytest.mark.parametrize("kind", ORCHESTRATORS) + def test_inactive_function_gives_no_rows(self, kind: str) -> None: + @function_pod(output_keys="y") + def inactive_inc(x: int) -> int: + return x + 1 + + db = InMemoryArrowDatabase() + _single(inactive_inc, ROWS[3:], db).run(orchestrator=_orchestrator(kind)) + # A function switched off produces no output for the inputs it is given. + inactive_inc.pod.data_function.set_active(False) + job = _single(inactive_inc, ROWS[:3], db) + job.run(orchestrator=_orchestrator(kind)) + node = job.nodes["node"] + assert node.as_table().num_rows == 0 + assert list(node.iter_data()) == [] + assert node.head(100).as_table().num_rows == 0 + + +class TestRaisedRun: + """A run that raised defines no rows: the node reads as a node with no run.""" + + @staticmethod + def _no_run_read(fn: Any, data: list[dict[str, int]], db: Any) -> list[Any]: + fresh = _single(fn, data, db) + fresh.compile() + return _values(fresh.nodes["node"], lambda r: (r["k"], r["y"])) + + @pytest.mark.parametrize( + ("kind", "fn", "policy"), + [ + ("sync", fails_on_20, "fail_fast"), + ("sync", store_fails_on_20, "continue"), + ("async", store_fails_on_20, "continue"), + ], + ) + def test_raised_run_reads_as_no_run(self, kind: str, fn: Any, policy: str) -> None: + db = InMemoryArrowDatabase() + # Store the results of rows 3..5 first, then run rows 0..3, which fails at x=20. + _single(fn, ROWS[3:], db).run(orchestrator=_orchestrator(kind)) + job = _single(fn, ROWS[:4], db) + with pytest.raises((ValueError, StoreWriteError)): + job.run(orchestrator=_orchestrator(kind, policy)) + node = job.nodes["node"] + expected = self._no_run_read(fn, ROWS[:4], db) + assert _values(node, lambda r: (r["k"], r["y"])) == expected + reads = _reads(node) + assert reads["head"] == reads["as_table"] + # Nothing stored was dropped: the stored rows of the earlier run are among them. + assert {(3, 31), (4, 41), (5, 51)} <= set(expected) From 2e9972e0ca0e2250d8c17c36f847d78869088968 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 20:51:24 -0800 Subject: [PATCH 02/48] fix(function_node): after a run a job node returns exactly the rows of that run's inputs (ITL-1093) The node records the base entry ids of its last completed run's inputs and every read of its rows goes through one helper that keeps only those inputs' rows. The async run loads every stored record into the cache, so it returned every row ever stored for the node; a re-run of the same node objects did so on the sync path too. A run that raised defines no rows. Nothing stored, no hash, no layout changes. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 7 +- CLAUDE.md | 8 +- changelog.d/ITL-1093.md | 3 + design_issues/ITL-1093-JR1.md | 26 ++++ docs/concepts/function-pods.md | 12 +- src/orcapod/core/nodes/function_node.py | 136 +++++++++++++----- .../test_job_node_rows_after_run.py | 99 +++++++++++-- 7 files changed, 236 insertions(+), 55 deletions(-) create mode 100644 changelog.d/ITL-1093.md create mode 100644 design_issues/ITL-1093-JR1.md diff --git a/.zed/rules b/.zed/rules index 0342f7342..2d923a544 100644 --- a/.zed/rules +++ b/.zed/rules @@ -773,8 +773,11 @@ data. The rules: by the user is never changed. SourceStreamBuilder.build has no bare-hash default: it needs a source_id or a source_kind. - ArrowTableStream requires at least one data column; raises ValueError otherwise. -- FunctionNode Phase 1 returns ALL records in the shared pipeline_path DB table. - Phase 2 skips inputs whose hash is already in the DB. +- After a run, a FunctionJobNode returns exactly the rows of that run's inputs, sync or + async (ITL-1093; a run that raised defines no rows). Read without a completed run in this + process (a fresh or loaded job), it returns every record stored under its content hash for + the current data context, not only its current inputs. A run skips inputs whose entry is + already stored. - Empty data → ArrowTableSource raises ValueError("Table is empty"). - DerivedSource over a job node before run() → presents zero rows with the origin's columns (no computed records; no error), in the plain form and with tags_as_data. diff --git a/CLAUDE.md b/CLAUDE.md index 8164ba5f9..d075910b4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -814,9 +814,11 @@ data. The rules: by the user is never changed. `SourceStreamBuilder.build` has no bare-hash default: it needs a `source_id` or a `source_kind`. - `ArrowTableStream` requires at least one data column; raises `ValueError` otherwise. -- `FunctionNode.iter_data()` Phase 1 returns ALL records in the shared `pipeline_path` - DB table (not filtered to current inputs). Phase 2 skips inputs whose hash is already - in the DB. +- After a run, a `FunctionJobNode` returns exactly the rows of that run's inputs, sync or + async (ITL-1093; a run that raised defines no rows). Read without a completed run in this + process (a fresh or loaded job), it returns every record stored under its content hash for + the current data context, not only its current inputs. A run skips inputs whose entry is + already stored. - Empty data → `ArrowTableSource` raises `ValueError("Table is empty")`. - `DerivedSource` over a job node before `run()` → presents zero rows with the origin's columns (no computed records; no error), in the plain form and with `tags_as_data`. diff --git a/changelog.d/ITL-1093.md b/changelog.d/ITL-1093.md new file mode 100644 index 000000000..2e4421eb8 --- /dev/null +++ b/changelog.d/ITL-1093.md @@ -0,0 +1,3 @@ +### Fixed + +- After `job.run()` on either orchestrator, a function job node's `as_table()`, `iter_data()`, `head()` and `run()`, and a stream built over the node, return exactly the rows of that run's inputs (ITL-1093). Before, after an async run a node returned every row stored under its key: a pipeline whose source rows changed between runs under one `source_id` showed the results of earlier runs too (for a function over `group_by`, the partial groups of an earlier run next to the full ones; rows of source rows no longer present). The sync orchestrator did the same when one job object ran again with its source rebound. Rows sent downstream during the run and the `RunSummary` were already right. Results of other inputs stay stored and are reused when those inputs come back; nothing stored, no hash and no table layout changes. A run that raised defines no rows: the node then reads as a node with no run in this process. A node read with no finished run in this process (a job built but not run, or a loaded job) still returns every row stored under its key. diff --git a/design_issues/ITL-1093-JR1.md b/design_issues/ITL-1093-JR1.md new file mode 100644 index 000000000..e270f4ffc --- /dev/null +++ b/design_issues/ITL-1093-JR1.md @@ -0,0 +1,26 @@ +## `src/orcapod/core/nodes/function_node.py` + +### JR1 — After a run, a function job node returned rows of other runs' inputs +**Status:** resolved +**Severity:** high +**Issue:** ITL-1093 + +A `FunctionJobNode` kept one in-memory cache of results keyed by base entry id, and every read +(`iter_data`, and through it `as_table`, `run()` and any consumer; `head`) returned all of it. +Nothing recorded which inputs the run processed. `async_execute` loads every pipeline record +stored under the node's content hash into that cache before routing its inputs, so after an +async run the node returned every row ever stored for it: partial groups of an earlier run next +to full ones, rows of source rows no longer present. The sync `execute` loaded only its inputs' +entries, so it looked right with a new job per run, but a cache is add-only: running the same +node objects again (one job, its source rebound) accumulated rows on the sync path too. Rows sent +downstream during the run and the `RunSummary` were right on both paths. + +**Fix:** The node records the base entry ids of its last completed run's inputs +(`_run_entry_counts`, set by `execute` and `async_execute`), and every read of its rows goes +through one helper, `_held_pairs`, which keeps only those inputs' rows (one per input, so two +equal inputs give two rows, as downstream received them). The cache stays a superset: stored +results of other inputs stay stored and reusable. A run that raised defines no rows: the node +clears its in-memory rows and reads as a node with no run in this process (every row stored +under its content hash), on both orchestrators. Nothing stored, no hash and no layout changed. +A read with no completed run in this process (a fresh or loaded job) still returns every row +stored under the node's content hash. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 108453b24..5f414b039 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -333,6 +333,13 @@ database. Each recorded step becomes a node, available afterwards as `job.nodes[ listed under its `source_id`). A node is a stream, so `as_table()`, `iter_data()` and `output_schema()` all work on it. +After `job.run()`, a function node returns exactly the rows of that run's inputs, on either +orchestrator. The database may hold more under the same node: results of an earlier run +over other rows of the source stay stored and are reused when those rows come back, but +they are not rows of this run. A run that raised defines no rows. A node read with no +finished run in this process (a job built but not run, or a loaded job) returns every row +stored under its key. + `InMemoryArrowDatabase` lasts only as long as the Python process. Use `DeltaTableDatabase("some/directory")` from `orcapod.databases` to keep results on disk between sessions. @@ -696,8 +703,9 @@ file names each run's rows by the content hash the run stored them under, so a l reads exactly what the same node built from your code would read from that store, and no rows under that key is an empty result, not an error. -- A function node's `as_table()`, `iter_data()` and `get_all_records()` return the rows the - saved run stored, with the types they were stored with. Rows come in the order the store +- A function node's `as_table()`, `iter_data()` and `get_all_records()` return the rows + stored under the saved run's key, with the types they were stored with: those of every + run of that node over the same sources, not only the saved run's inputs. Rows come in the order the store returns them; sort them if order matters. - Operators (a join, a filter) store outputs only when caching is enabled, under `odb_v1`. The saved file does not record an operator's cache mode (follow-up ITL-977), so attach a diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 186a8e384..aa9c50dff 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -21,7 +21,8 @@ import logging import uuid from abc import abstractmethod -from collections.abc import Callable, Iterator, Mapping, Sequence +from collections import Counter +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Literal, NamedTuple, cast from orcapod import contexts @@ -1054,6 +1055,10 @@ class FunctionJobNode(FunctionNodeBase): _current_tables_checked: bool = False # Set once a store that cannot list its tables was reported. _unlisted_store_reported: bool = False + # Base entry ids of the inputs of the last completed run in this process, each with + # its number of inputs; None when no run completed (ITL-1093). A class default so + # nodes rebuilt from a saved descriptor start without a run. + _run_entry_counts: Counter[bytes] | None = None def __init__( self, @@ -1083,6 +1088,7 @@ def __init__( ] = {} self._cached_output_table: "pa.Table | None" = None self._cached_content_hash_column: "pa.Array | None" = None + self._run_entry_counts = None # DB persistence state (initially None; set via __init__ params or attach_databases) self._pipeline_database: ArrowDatabaseProtocol | None = None @@ -1505,11 +1511,44 @@ def from_descriptor( # ------------------------------------------------------------------ def clear_cache(self) -> None: - """Clear in-memory output caches, content hash cache, and node identity path cache.""" + """Clear in-memory output caches, content hash cache, and node identity path cache. + + The node then holds no run: it reads as a node with no completed run in this + process (every row stored under its content hash). + """ super().clear_cache() self._cached_output_datas.clear() self._cached_output_table = None self._cached_content_hash_column = None + self._run_entry_counts = None + + def _held_pairs(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: + """Yield the (tag, data) rows this node holds; every read of them goes through here. + + After a completed run, the rows are those of that run's inputs: one per input + (two equal inputs give two rows, as downstream received them). The in-memory + cache may hold more (results of other inputs, read from the store), which stay + cached for reuse but are not rows of this run (ITL-1093). Without a completed + run in this process, every held row. Rows come in the order the cache holds + them. An input without output (an inactive function) is not a row. + """ + held = self._cached_output_datas + counts = self._run_entry_counts + if counts is None: + pairs: Iterable[tuple[TagProtocol, DataProtocol | None]] = held.values() + else: + pairs = ( + pair for eid, pair in held.items() for _ in range(counts.get(eid, 0)) + ) + for tag, data in pairs: + if data is not None: + yield tag, data + + def _set_run_rows(self, entry_ids: list[bytes]) -> None: + """Make the inputs of a completed run, by base entry id, the rows this node holds.""" + self._run_entry_counts = Counter(entry_ids) + self._cached_output_table = None + self._cached_content_hash_column = None def set_ephemeral_store(self, store: "ArrowDatabaseProtocol | None") -> None: """Assign or remove the ephemeral result store. @@ -1872,6 +1911,11 @@ def execute( (populated from DB if needed), compute if missing, fire ``on_data_end`` or ``on_data_crash``. + Afterwards the node's rows (``iter_data``, ``as_table``, ``head``) are those + of this run's inputs. If the run raises, it defines no rows: the node drops + its in-memory rows and reads as a node with no run in this process; what was + stored stays stored. + Args: input_stream: The input stream to process. observer: Optional execution observer for hooks. @@ -1910,23 +1954,24 @@ def execute( tag_schema = input_stream.output_schema(columns={"system_tags": True})[0] ctx_obs.on_node_start(node_label, node_hash, tag_schema=tag_schema) - # Collect upstream entries and resolve base_entry_ids (stable across recomputation) - upstream_entries: list[tuple[TagProtocol, DataProtocol, bytes]] = [ - (tag, data, self.compute_base_entry_id(tag, data)) - for tag, data in input_stream.iter_data() - ] - base_entry_ids = [eid for _, _, eid in upstream_entries] - - # Hot-load any already-computed results from DB into _cached_output_datas. - # get_cached_results() is called for its side effect (populating the - # in-memory cache); the returned dict is intentionally discarded here so - # that the per-data cache-hit check below uses _cached_output_datas - # directly — which includes None-output entries (function returned None) - # and prevents spurious recomputation of already-processed data. - self.get_cached_results(base_entry_ids=base_entry_ids) - output: list[tuple[TagProtocol, DataProtocol]] = [] try: + # Collect upstream entries and resolve base_entry_ids (stable across + # recomputation) + upstream_entries: list[tuple[TagProtocol, DataProtocol, bytes]] = [ + (tag, data, self.compute_base_entry_id(tag, data)) + for tag, data in input_stream.iter_data() + ] + base_entry_ids = [eid for _, _, eid in upstream_entries] + + # Hot-load any already-computed results from DB into _cached_output_datas. + # get_cached_results() is called for its side effect (populating the + # in-memory cache); the returned dict is intentionally discarded here so + # that the per-data cache-hit check below uses _cached_output_datas + # directly — which includes None-output entries (function returned None) + # and prevents spurious recomputation of already-processed data. + self.get_cached_results(base_entry_ids=base_entry_ids) + for tag, data, base_entry_id in upstream_entries: ctx_obs.on_data_start(node_label, tag, data) @@ -1993,8 +2038,15 @@ def execute( ) except Exception as run_error: self._commit_after_failure(run_error) + # A run that raised defines no rows: the node reads as one with no run. + self.clear_cache() + raise + except BaseException: + self.clear_cache() raise + # The node's rows are those of this run's inputs, whatever else it caches. + self._set_run_rows(base_entry_ids) ctx_obs.on_node_end(node_label, node_hash) # Mark this node as freshly computed so subsequent iter_data() calls # skip the is_stale check and serve results directly from the in-memory cache. @@ -3086,8 +3138,8 @@ def head( ) -> StreamView: """Return the first rows the node holds or has stored; never runs the function. - After a run (or ``execute``) the rows are the node's results in memory, the - first of what ``as_table`` returns. Otherwise they are the first of its stored + After a run (or ``execute``) the rows are the node's results in memory for that + run's inputs, the first of what ``as_table`` returns. Otherwise they are the first of its stored rows: the node's whole stored slice is read and nothing is kept on the node. With nothing stored, there are no rows. The node has all of its rows, so with ``sort_by_tags`` (or ``all_info``) the head is exactly @@ -3107,9 +3159,11 @@ def head( config, ViewStatus.UNAVAILABLE, note="no store attached" ) current = status == LoadStatus.CACHE_ONLY or not self.is_stale - if current and self._cached_output_datas: + if current and ( + self._run_entry_counts is not None or self._cached_output_datas + ): return self._head_of_pairs( - list(self._cached_output_datas.values()), n, config, origin="held" + list(self._held_pairs()), n, config, origin="held" ) return self._head_from_store(n, config) @@ -3197,13 +3251,17 @@ def as_table( return super().as_table(columns=columns, all_info=all_info) def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: - """Yield all computed (tag, data) pairs for this node. + """Yield this node's (tag, data) rows. Strictly read-only — never triggers computation. Callers must call ``run()`` or ``execute()`` first if they want results computed. - On the first call with an empty in-memory store and a DB attached, - hot-loads all existing records from the DB (one-shot, no recompute). + After a completed run (``execute``, ``async_execute``, ``run()`` or a job run) + the rows are exactly those of that run's inputs, on either orchestrator, even + though results of other inputs stay cached and stored (ITL-1093). A run that + raised defines no rows. Without a completed run in this process, on the first + call with an empty in-memory store and a DB attached, hot-loads every record + stored under the node's content hash (one-shot, no recompute) and yields them. Raises: RuntimeError: If ``load_status`` is UNAVAILABLE. @@ -3222,11 +3280,7 @@ def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: if loaded: self._cached_output_table = None self._cached_content_hash_column = None - yield from ( - (tag, pkt) - for tag, pkt in self._cached_output_datas.values() - if pkt is not None - ) + yield from self._held_pairs() return # FULL / READ_ONLY — in-memory store may be populated from computation @@ -3245,11 +3299,7 @@ def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: self._cached_output_table = None self._cached_content_hash_column = None - yield from ( - (tag, pkt) - for tag, pkt in self._cached_output_datas.values() - if pkt is not None - ) + yield from self._held_pairs() def run(self) -> StreamView: """Eagerly compute all input data, filling pipeline and result databases, and @@ -3312,6 +3362,11 @@ async def async_execute( are served directly from persistent storage without touching the input channel. + Afterwards the node's rows are those of this run's inputs, as after + ``execute``, although every stored result was loaded into its cache. If the + run raises or is cancelled, it defines no rows: the node drops its in-memory + rows and reads as a node with no run in this process. + Args: input_channel: Single readable channel of (tag, data) pairs. output: Writable channel for output (tag, data) pairs. @@ -3405,12 +3460,15 @@ async def async_execute( # Local dict: correlation_key → (original_tag, original_input_data) input_store: dict[bytes, tuple[TagProtocol, DataProtocol, bytes]] = {} + # Base entry ids of this run's inputs, in arrival order (ITL-1093). + run_entry_ids: list[bytes] = [] async def route_inputs() -> None: """Stage 1: send cache hits to output; stamp misses for computation.""" try: async for tag, data in input_channel: base_entry_id = self.compute_base_entry_id(tag, data) + run_entry_ids.append(base_entry_id) # A cached EmptyData is a result that is gone (an # ephemeral result from an earlier run): compute the row # again, as the sync path does (ITL-923). @@ -3589,6 +3647,9 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: tg.create_task(record_and_forward()) # Commit the last rows inside a job (ITL-919); a no-op otherwise. self._commit_at_node_end() + # The node's rows are those of this run's inputs, as the sync path's: + # the cache also holds every stored result loaded above. + self._set_run_rows(run_entry_ids) else: # No-DB path: delegate dispatch entirely to the pod. The pod closes # the writer it is given; that close waits for the finally below, so @@ -3607,6 +3668,13 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: self._update_modified_time() except Exception as run_error: self._commit_after_failure(run_error) + # A run that raised defines no rows: the node reads as one with no run, + # as on the sync path. + self.clear_cache() + raise + except BaseException: + # Cancelled part-way: no rows either. + self.clear_cache() raise finally: await output.close() diff --git a/tests/test_pipeline/test_job_node_rows_after_run.py b/tests/test_pipeline/test_job_node_rows_after_run.py index 7f2ac88c8..fd57128b0 100644 --- a/tests/test_pipeline/test_job_node_rows_after_run.py +++ b/tests/test_pipeline/test_job_node_rows_after_run.py @@ -17,6 +17,7 @@ import pytest from orcapod import PipelineJob, function_pod +from orcapod.core.data_function import PythonDataFunction from orcapod.databases import InMemoryArrowDatabase from orcapod.errors import StoreWriteError from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator @@ -63,7 +64,7 @@ def store_fails_on_20(x: int) -> int: failed write of the node's results does. """ if x == 20: - raise StoreWriteError("simulated failed store write") + raise StoreWriteError(("simulated", "results")) return x + 1 @@ -104,7 +105,8 @@ def _canonical(rows: list[dict[str, Any]]) -> list[tuple[tuple[str, str], ...]]: def _reads(node: Any) -> dict[str, list[tuple[tuple[str, str], ...]]]: """Every public read of ``node``, rows with system tags, canonical.""" iterated = [ - {**tag.as_dict(all_info=True), **data.as_dict()} for tag, data in node.iter_data() + {**tag.as_dict(all_info=True), **data.as_dict()} + for tag, data in node.iter_data() ] return { "as_table": _canonical(node.as_table(columns=ALL).to_pylist()), @@ -161,7 +163,10 @@ def _check_run(job: PipelineJob, data: list[dict[str, int]]) -> None: def _snapshot(job: PipelineJob) -> dict[str, Any]: - return {label: _reads(job.nodes[label]) for label in ("rowwise", "downstream", "grouped")} + return { + label: _reads(job.nodes[label]) + for label in ("rowwise", "downstream", "grouped") + } class TestFreshJobPerRun: @@ -227,10 +232,10 @@ def test_sync_and_async_agree_row_for_row(self) -> None: assert snapshots["sync"] == snapshots["async"] -def _single(fn: Any, data: list[dict[str, int]], db: Any) -> PipelineJob: +def _single(pod: Any, data: list[dict[str, int]], db: Any) -> PipelineJob: job = PipelineJob(name="itl1093_single", database=db) with job: - fn.pod(_source(data), label="node") + pod(_source(data), label="node") return job @@ -240,8 +245,8 @@ class TestNullAndInactive: @pytest.mark.parametrize("kind", ORCHESTRATORS) def test_null_value_is_a_row(self, kind: str) -> None: db = InMemoryArrowDatabase() - _single(null_on_10, ROWS, db).run(orchestrator=_orchestrator(kind)) - job = _single(null_on_10, ROWS[:3], db) + _single(null_on_10.pod, ROWS, db).run(orchestrator=_orchestrator(kind)) + job = _single(null_on_10.pod, ROWS[:3], db) job.run(orchestrator=_orchestrator(kind)) node = job.nodes["node"] assert _values(node, lambda r: (r["k"], r["y"])) == [(0, 1), (1, None), (2, 21)] @@ -252,14 +257,16 @@ def test_null_value_is_a_row(self, kind: str) -> None: @pytest.mark.parametrize("kind", ORCHESTRATORS) def test_inactive_function_gives_no_rows(self, kind: str) -> None: @function_pod(output_keys="y") - def inactive_inc(x: int) -> int: + def add_one(x: int) -> int: return x + 1 db = InMemoryArrowDatabase() - _single(inactive_inc, ROWS[3:], db).run(orchestrator=_orchestrator(kind)) + _single(add_one.pod, ROWS[3:], db).run(orchestrator=_orchestrator(kind)) # A function switched off produces no output for the inputs it is given. - inactive_inc.pod.data_function.set_active(False) - job = _single(inactive_inc, ROWS[:3], db) + data_function: object = add_one.pod.data_function + assert isinstance(data_function, PythonDataFunction) + data_function.set_active(False) + job = _single(add_one.pod, ROWS[:3], db) job.run(orchestrator=_orchestrator(kind)) node = job.nodes["node"] assert node.as_table().num_rows == 0 @@ -272,7 +279,7 @@ class TestRaisedRun: @staticmethod def _no_run_read(fn: Any, data: list[dict[str, int]], db: Any) -> list[Any]: - fresh = _single(fn, data, db) + fresh = _single(fn.pod, data, db) fresh.compile() return _values(fresh.nodes["node"], lambda r: (r["k"], r["y"])) @@ -287,8 +294,8 @@ def _no_run_read(fn: Any, data: list[dict[str, int]], db: Any) -> list[Any]: def test_raised_run_reads_as_no_run(self, kind: str, fn: Any, policy: str) -> None: db = InMemoryArrowDatabase() # Store the results of rows 3..5 first, then run rows 0..3, which fails at x=20. - _single(fn, ROWS[3:], db).run(orchestrator=_orchestrator(kind)) - job = _single(fn, ROWS[:4], db) + _single(fn.pod, ROWS[3:], db).run(orchestrator=_orchestrator(kind)) + job = _single(fn.pod, ROWS[:4], db) with pytest.raises((ValueError, StoreWriteError)): job.run(orchestrator=_orchestrator(kind, policy)) node = job.nodes["node"] @@ -298,3 +305,67 @@ def test_raised_run_reads_as_no_run(self, kind: str, fn: Any, policy: str) -> No assert reads["head"] == reads["as_table"] # Nothing stored was dropped: the stored rows of the earlier run are among them. assert {(3, 31), (4, 41), (5, 51)} <= set(expected) + + +class TestDuplicateInputs: + """Two inputs with one base entry id give two held rows, as downstream receives them. + + Only a stream without system tags can carry two such inputs (a source gives every + row its own record id), so the node is built directly over an ``ArrowTableStream``. + """ + + @staticmethod + def _node() -> Any: + import pyarrow as pa + + from orcapod.core.nodes import FunctionJobNode + from orcapod.core.streams import ArrowTableStream + + schema = pa.schema( + [ + pa.field("k", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] + ) + table = pa.Table.from_arrays( + [pa.array([0, 0, 2], pa.int64()), pa.array([5, 5, 7], pa.int64())], + schema=schema, + ) + return FunctionJobNode( + inc.pod, # pyright: ignore[reportArgumentType] # the decorator's pod type is not declared as FunctionPodProtocol + ArrowTableStream(table, tag_columns=["k"]), + pipeline_database=InMemoryArrowDatabase(), + result_database=InMemoryArrowDatabase(), + ) + + def test_sync(self) -> None: + node = self._node() + view = node.run() + held = sorted((r["k"], r["y"]) for r in node.as_table().to_pylist()) + assert held == [(0, 6), (0, 6), (2, 8)] + assert sorted((r["k"], r["y"]) for r in view.as_table().to_pylist()) == held + + def test_async(self) -> None: + import asyncio + + from orcapod.channels import Channel + + node = self._node() + + async def run() -> list[tuple[int, int]]: + inputs = Channel(buffer_size=16) + outputs = Channel(buffer_size=16) + # The input stream is the node's own upstream (a test of async_execute). + for item in node.upstreams[0].iter_data(): + await inputs.writer.send(item) + await inputs.writer.close() + await node.async_execute(inputs.reader, outputs.writer) + got = [ + (tag.as_dict()["k"], data.as_dict()["y"]) + async for tag, data in outputs.reader + ] + return sorted(got) + + received = asyncio.run(run()) + assert received == [(0, 6), (0, 6), (2, 8)] + assert sorted((r["k"], r["y"]) for r in node.as_table().to_pylist()) == received From 51cb1b5c90963a876ea2376fe3dcde9d6e7f22fd Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:20:37 -0800 Subject: [PATCH 03/48] docs(plans): ITL-1095 diagnosis and proposal (WIP) Co-Authored-By: Claude Opus 5.5 --- ...26-11-13-itl-1095-node-current-run-view.md | 172 +++++++++++++++ .../plans/2026-11-13-itl-1095-proposal.md | 196 ++++++++++++++++++ 2 files changed, 368 insertions(+) create mode 100644 superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md create mode 100644 superpowers/plans/2026-11-13-itl-1095-proposal.md diff --git a/superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md b/superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md new file mode 100644 index 000000000..de97f4096 --- /dev/null +++ b/superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md @@ -0,0 +1,172 @@ +# ITL-1095: a job node shows the current run; load-only run + +Status: DIAGNOSIS + PROPOSAL sent to the coordinator (ask), waiting for decisions. No code changed. +Base: PR #378 branch head `2e9972e0` (ITL-1093). Branch: `eywalker/itl-1095-design-what-a-job-node-shows-the-current`. + +Rule (developer, November 13, 2026): a job node itself (`as_table`, `iter_data`, dataframe +methods, `flow`, `head`, the preview, `run`) is the CURRENT RUN view; `node.as_source()` is the +view of everything stored for the node. Before a run a node is empty; a run fills it, also when +everything is loaded. `run(load_only=True)` fills nodes from what is stored and computes nothing. +`node.run()` cascades to the job nodes it depends on (developer, later the same day). + +## 1. Measurements on the base (`2e9972e0`) + +Scratch scripts: `measure.py`, `replay.py`, `noderun.py` in the session scratchpad. +Pipeline: `DictSource(source_id="s", tags g,k)` -> `inc` (`rowwise`) -> `group_by("g")` +(`grouped`, operator) -> `total` (`summed`). Runs over 6 rows then 3 rows, one Delta store. + +| Read | sync | async | +|---|---|---| +| after run over 3 rows: rowwise / grouped / summed `as_table` | 3 / 2 / 2 | 3 / 2 / 2 | +| `as_source()` of rowwise / summed | 6 / 4 | 6 / 4 | +| fresh job (3 source rows), NOT run: rowwise / grouped(OFF) / summed `as_table` | **6** / 0 / **4** | same | +| fresh job, not run: `head()` rowwise | `rows`, 6 of 6 (stored) | same | +| fresh job: `summed.run()` alone (upstream OFF operator not run) | 0 rows, nothing computed | - | +| fresh job: source node `as_table` | 3 (bound source) | same | + +Operator node `grouped` before a run (fresh job over a store with two earlier runs): + +| cache mode | `as_table` | `head` | `as_source` | +|---|---|---|---| +| OFF | 0 | not_computed "not run; cache mode OFF stores nothing" | 0 | +| LOG | 0 | not_computed "rows of earlier runs are stored but not replayed" | 4 (both runs) | +| REPLAY (after two LOG runs) | **4** (both runs) | rows 4 | 4 | + +REPLAY after a run over 3 rows (sync and async alike): `grouped` holds **4 groups** +`(0,[1,21]) (0,[1,21,41]) (1,[11]) (1,[11,31,51])` -- every logged run, not the current run -- +and `summed` downstream gets 4 rows. The operator cache (`odb_v1`) stores no run id or time, so +"the latest logged run" cannot be told apart without a new stored column. + +`node.run()` with an upstream job node that has not run (chain `a` -> `b`, store holds a 6-row run, +source now 3 rows): `b.run()` reads `a`'s STORED rows (6, not the current 3) and returns 6 rows, +0 function calls; over an empty store it returns 0 rows silently. `node.run()` never runs upstream. + +Loaded job (`PipelineJob.load`, in-memory source unbound): function nodes are `READ_ONLY` stubs +showing every stored row (rowwise 6, summed 4); the OFF operator raises `RuntimeError` from +`as_table` (head: unavailable); source node raises `UnboundSourceError`; `loaded.run()` raises. +Loaded `rowwise.as_source()` reads 6 rows. + +Load statuses: a built job's function nodes are always `FULL`. A loaded node without a pod is +`READ_ONLY` (store attached) or `UNAVAILABLE`. `CACHE_ONLY` and `READ_ONLY` with a +`DataFunctionProxy` arise only from `FunctionJobNode.from_descriptor(..., function_pod=...)`, +which no public path calls (`PipelineJob.load` passes no pod). + +Orchestrators: both pass rows between nodes in their own buffers/channels; no node read is +involved in `job.run()`. Only `node.run()` reads its upstream node. + +Experiment (reverted): making every un-run read empty fails 400 of 4,464 job-related tests in +21 files (list in section 5). + +## 2. Proposal + +See the ask message (copied in section 6 once answered). + +## 3. Implementation sketch (after approval) + +- `FunctionJobNode`: no-run read empty (all load statuses); `head` NOT_COMPUTED with note; + `execute`/`async_execute` take `load_only` (lookup `get_cached_results` only, skip misses, count + them, no `_process_data_internal`, no writes); `_last_run_skipped` count; run mode flag. +- `OperatorJobNode`: REPLAY before run empty; load-only: OFF/LOG run the operator, LOG writes + nothing; REPLAY replays. +- Orchestrators: `load_only` threaded through; side-effect node passes rows through uncalled. +- `PipelineJob.run/run_async(*, load_only=False)`; loaded job allowed with `load_only=True`. +- One internal "run these job nodes" function shared by `job.run` and the `node.run` cascade. +- `RunSummary`/`NodeRunSummary` new field(s). +- Tests first (public interface, both orchestrators, 3.11 for async). +- Docs, changelog fragment, design_issues entry, CLAUDE.md + .zed/rules. + +## 5. Tests and docs that change + +Failing under the experiment (count per file): test_node_head_sorting 162, +test_loaded_operator_cache 74, test_function_node_arrow_types 58, test_loaded_job_readback 29, +test_delta_arrow_type_roundtrip 25, test_node_head 9, test_loaded_logical_schemas 7, +test_provisional_hash_warning 6, test_function_pod_node_stream 5, test_stored_data_context 4, +test_function_node_empty_data 4, test_job_node_rows_after_run 3, test_tensor_job_load_back 2, +test_materialize_source_info 2, test_function_node_caching 2, test_cached_function_pod_job 2, +test_doc_samples (function-pods.md) 1, test_operator_node 1, test_operator_node_non_active 1, +test_function_node_iteration 1, test_readback_datagram_id 1, test_polars_dataframe_values 1. +Plus `tests/test_core/streams/test_run.py::test_every_stream_class_has_run_without_arguments` +(a `load_only` keyword on a node's `run`). Full names: scratchpad `hack_failed_tests.txt`. + +Docs: docs/concepts/streams.md (run table row "A job node", head table row "A job node", the +"stream that has all of its rows" paragraph, line ~262), docs/concepts/function-pods.md (339-341 +no-run read; "Saving and loading a job" 662-746 and its sample at 689-694, the only doc sample +that reads a node without a run), docs/concepts/sources.md 169-172; docstrings of +`PipelineJob.load`/`run`, `FunctionJobNode.head/as_table/iter_data/run/clear_cache/_cannot_compute_error`, +`OperatorJobNode.iter_data/as_table/head`, `ViewStatus.NOTHING_STORED`, `StreamBase.head`. + +### Appendix: tests failing under the empty-before-run experiment (parametrizations collapsed) + +``` +tests/test_core/function_pod/test_cached_function_pod_job.py::test_saved_job_equals_the_uncached_pods_and_loads +tests/test_core/function_pod/test_function_node_arrow_types.py::test_store_read_back_keeps_tag_type +tests/test_core/function_pod/test_function_node_caching.py::TestPhase1Phase2PipelineEntryId::test_all_existing_entry_ids_skipped_in_phase2 +tests/test_core/function_pod/test_function_node_caching.py::TestPhase1Phase2PipelineEntryId::test_phase1_yields_existing_results +tests/test_core/function_pod/test_function_pod_node_stream.py::TestFunctionNodeInactive::test_as_table_returns_cached_results_when_data_function_inactive +tests/test_core/function_pod/test_function_pod_node_stream.py::TestFunctionNodeInactive::test_inactive_filled_db_serves_cached_results +tests/test_core/function_pod/test_function_pod_node_stream.py::TestFunctionNodeStaleness::test_as_table_auto_detects_stale_and_repopulates +tests/test_core/function_pod/test_function_pod_node_stream.py::TestFunctionNodeStaleness::test_clear_cache_produces_same_results_on_re_iteration +tests/test_core/function_pod/test_function_pod_node_stream.py::TestIterDatasDbPhase::test_db_served_results_have_correct_row_count +tests/test_core/function_pod/test_polars_dataframe_values.py::TestPipelineStores::test_saved_job_loaded_back_reads_the_frames +tests/test_core/function_pod/test_readback_datagram_id.py::test_a_fresh_job_on_the_store_reads_the_stored_ids +tests/test_core/nodes/test_function_node_empty_data.py::TestAsyncProcessDataInternalEmptyData::test_async_empty_data_cache_hit_returns_cached_result +tests/test_core/nodes/test_function_node_empty_data.py::TestAsyncProcessDataInternalEmptyData::test_async_empty_data_cache_miss_raises +tests/test_core/nodes/test_function_node_empty_data.py::TestProcessDataInternalEmptyData::test_cache_hit_returns_cached_result +tests/test_core/nodes/test_function_node_empty_data.py::TestProcessDataInternalEmptyData::test_cache_miss_raises_ephemeral_result_missing_error +tests/test_core/nodes/test_function_node_iteration.py::TestIterDatasReadOnly::test_iter_after_db_populated_hot_loads_without_compute +tests/test_core/operators/test_materialize_source_info.py::TestStoreWrittenBeforeFix::test_rerun_after_pre_fix_store +tests/test_core/operators/test_materialize_source_info.py::TestStoreWrittenBeforeFix::test_v014_store_is_not_read +tests/test_core/operators/test_operator_node_non_active.py::TestCacheModeNonActive::test_iter_data_replay_mode_returns_db_contents +tests/test_core/operators/test_operator_node.py::TestOperatorNodeRunAndStorage::test_replay_from_cache +tests/test_databases/test_delta_arrow_type_roundtrip.py::test_a_fresh_job_reads_the_tag_type +tests/test_databases/test_delta_arrow_type_roundtrip.py::test_a_loaded_job_reads_the_tag_type +tests/test_databases/test_delta_arrow_type_roundtrip.py::test_a_rerun_on_the_same_store_writes_no_pipeline_rows +tests/test_databases/test_delta_arrow_type_roundtrip.py::test_an_operator_replay_reads_the_tag_type +tests/test_docs/test_doc_samples.py::test_doc_page_samples_run +tests/test_logical_types/test_provisional_hash_warning.py::test_reading_stored_results_of_the_type_warns_once +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_binding_a_store_makes_a_storeless_loaded_node_readable +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_datetime_output_feeding_an_operator_loads_with_its_type +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_file_without_the_expected_rows_section_reads_without_a_check +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_latest_readable_result_wins_across_both_stores +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_live_node_hot_load_reads_the_latest_recomputed_result +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_loaded_function_node_as_table_all_info_has_stream_columns +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_loaded_function_node_as_table_returns_stored_rows +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_loaded_function_node_iter_data_yields_stored_rows +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_loaded_node_with_ephemeral_results_does_not_crash +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_loaded_operator_node_raises_instead_of_returning_empty +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_logical_type_output_schema_fails_naming_the_column_and_saved_type +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_newer_unavailable_result_does_not_expose_old_persistent_result +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_other_context_records_do_not_satisfy_saved_expectation +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_path_output_feeding_join_still_allows_job_load +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_recomputed_entry_is_read_once_with_its_latest_result +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_saving_a_loaded_job_keeps_the_expected_rows +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_store_content_warning_names_a_delta_stores_location +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_store_with_only_another_runs_records_reads_none_with_a_warning +tests/test_pipeline/loaded_job/test_loaded_job_readback.py::test_unreconstructable_file_source_is_a_warning_with_the_reason +tests/test_pipeline/loaded_job/test_loaded_logical_schemas.py::test_registered_logical_output_keeps_join_available +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_a_format_0_1_0_file_finds_its_join_table_and_never_shows_its_rows_as_none +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_a_loaded_run_with_an_empty_result_reads_as_empty +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_a_pipeline_loaded_and_bound_to_its_sources_reuses_the_operator_cache +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_a_store_holding_only_other_runs_rows_gives_an_empty_result +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_an_absent_cache_table_gives_an_empty_result +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_fewer_rows_than_expected_warn_once_and_read_what_the_store_holds +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_get_cached_output_of_a_loaded_node_never_caches_unreadable_rows_as_none +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_loaded_replay_operator_reads_the_rows_its_run_stored +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_matching_counts_and_zero_expected_zero_found_do_not_warn +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_more_rows_than_expected_warn_and_read_them_all +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_without_saved_nullability_the_operator_still_reads_its_runs_table +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_without_the_run_content_hash_an_absent_table_raises_too +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_without_the_run_content_hash_head_says_why_it_shows_no_rows +tests/test_pipeline/loaded_job/test_loaded_operator_cache.py::test_without_the_run_content_hash_the_loaded_operator_raises +tests/test_pipeline/test_job_node_rows_after_run.py::TestRaisedRun::test_raised_run_reads_as_no_run +tests/test_pipeline/test_node_head_sorting.py::test_a_function_nodes_head_is_the_prefix_of_as_table_in_any_completion_order +tests/test_pipeline/test_node_head_sorting.py::test_an_operator_nodes_head_is_the_prefix_of_as_table +tests/test_pipeline/test_node_head.py::test_a_loaded_job_shows_stored_rows_and_never_raises +tests/test_pipeline/test_node_head.py::test_a_new_job_over_a_filled_store_reads_stored_rows_without_computing +tests/test_pipeline/test_node_head.py::test_head_keeps_the_tag_arrow_type_of_the_input +tests/test_pipeline/test_node_head.py::test_nodes_before_a_run_show_no_rows_and_compute_nothing +tests/test_pipeline/test_node_head.py::test_operator_node_cache_modes +tests/test_pipeline/test_stored_data_context.py::TestRowsOfAnotherDataContext::test_function_node_reads_current_rows_only +tests/test_pipeline/test_stored_data_context.py::TestSideEffects::test_function_node_upstream_of_a_refused_sink_is_not_refused +tests/test_torch/test_tensor_job_load_back.py::test_a_loaded_job_reads_its_stored_tensors +``` diff --git a/superpowers/plans/2026-11-13-itl-1095-proposal.md b/superpowers/plans/2026-11-13-itl-1095-proposal.md new file mode 100644 index 000000000..4478eaed4 --- /dev/null +++ b/superpowers/plans/2026-11-13-itl-1095-proposal.md @@ -0,0 +1,196 @@ +# ITL-1095 proposal (sent to the coordinator for decisions) + +Measurements: `2026-11-13-itl-1095-node-current-run-view.md` section 1 (same folder). +Recommendations are marked **(rec)**. Decisions needed: D1 to D11. + +## A. The rule as built, per node kind + +| Node | Before a run in this process (every read) | After a run | +|---|---|---| +| `FunctionJobNode` (FULL; the only state a built job has) | zero rows, typed columns; `head`/preview status `not_computed`, note below | the run's rows (#378); a load-only run: the loaded rows | +| same, after a run that raised / `clear_cache()` / stale (upstream re-ran or was rebound) | empty, as before a run (today: every stored row) | - | +| loaded function node (`READ_ONLY` stub) | empty (today: every stored row) | after `loaded.run(load_only=True)`: loaded rows | +| `CACHE_ONLY`, `READ_ONLY` with a proxy pod (only via `from_descriptor(..., function_pod=...)`, no public path) | empty | D9 | +| `OperatorJobNode` OFF / LOG | empty (unchanged: already so) | run's output (unchanged) | +| `OperatorJobNode` REPLAY | empty (today: union of every logged run) | D1 | +| loaded operator node | empty, no raise (today OFF raises `RuntimeError`) | after a load-only run: its output on the loaded rows | +| `SideEffectJobNode` | #377 (not touched): reads deliver nothing and return zero rows when nothing is held. Already fits the rule. | #377 | +| `SourceJobNode` | the bound source's rows, before and after (a source is not computed); unbound raises as today. **Yes, unchanged.** | same | + +`node.as_source()` unchanged: every stored record of the node (current data context). + +Head/preview note before a run (D10) **(rec)**: status `not_computed`, header like +`FunctionJobNode[rowwise] — not run; run() fills it, run(load_only=True) loads stored results only; node.as_source() shows every stored result`. +The stored count is NOT available without reading rows (the database protocol has no count; today's +head reads the whole stored slice to show stored rows). Exception: a loaded job whose file records +`expected_rows` for the node: "the saved run stored N rows". **(rec)** no count otherwise, and +`ViewStatus.NOTHING_STORED` then has no producer: remove it (greenfield) **(rec)**, or keep. + +## B. Load-only run: `job.run(*, load_only=False)`, `job.run_async(*, load_only=False)`, `node.run(*, load_only=False)` + +Keyword-only, default False. + +- Sources are read. +- Function node: computes each input's entry id, looks up the node's OWN stored entries (pipeline + records under its content hash, the lookup a run does today), forwards only the rows it loaded, + skips and counts inputs without one. No user function call, no side effect, no write to result + or pipeline tables. An entry whose result was ephemeral and is gone counts as not stored. + - **D2**: an input with no pipeline record for this node but a matching result in the + function's result store (same function in another node, or a source renamed with the same + rows; a normal run "reuses" it and writes a pipeline record). (a) skip it, count it as not + stored **(rec)**: what the node shows is then always a subset of `as_source()`. (b) show it + without writing a record. +- Operator node, **D3**: (a) OFF and LOG run their operator on the rows that reach them (cheap, + not stored; LOG writes nothing in a load-only run), REPLAY replays as in a normal run + **(rec)**; (b) only cached operators produce rows: OFF (the only mode a job sets) forwards + nothing, so everything below any operator is empty: load-only becomes useless for + `group_by`, `join`, `select` pipelines. + - GroupBy hazard under (a): if the function node above lost inputs (not stored), the groups are + PARTIAL; a partial group's input differs from the full group's, so the function below misses + -- unless an earlier, smaller run stored exactly that partial group. Measured case: store has + the 3-row run, source now 6 rows, `rowwise` stored for 3 of 6 -> load-only groups 3 rows -> + `summed` HITS the 3-row run's totals (0,22),(1,11) and shows them as this run's. Options: + (i) the summary marks every node downstream of a node with skipped inputs as "inputs + incomplete (rowwise skipped 3)" and the docs say so **(rec for the tag)**; (ii) a node below + a node with skipped inputs forwards nothing (strict; also blanks row-wise chains, which are + exact); (iii) nothing. +- Side-effect node (D4): **(rec)** the orchestrator passes its input rows through without calling + it (no delivery, no log write; no change to `side_effects.py`, which #377 owns); the node then + holds nothing and reads empty. After #377 merges it could hold the passed rows instead. + A ctx_arg function node is a function node: loads, calls nothing. +- Both orchestrators: same per-node behaviour; `load_only` passed through `run`/`run_async` to + `execute`/`async_execute`. +- (d) `error_policy`: nothing to apply it to (no function runs); store read errors and + `TypeIdentityError` raise as in any run, operator errors raise as today. Observers: + `on_run_start/end`, `on_node_start/end` per node; a loaded row gets `on_data_start` + + `on_data_end(cached=True)`; a skipped input fires no row hook (no observer protocol change; + the node counts it and the job reads the count). A user-supplied status/logging observer writes + its own observability rows as in any run (not result/pipeline tables) -- say if load-only should + suppress them. +- **D5 RunSummary shape (rec)**: `NodeRunSummary` gains `not_stored: int = 0` (inputs skipped + because no result is stored, load-only only) and `inputs_incomplete: tuple[str, ...] = ()` + (upstream nodes that skipped inputs, D3(i)); loaded rows count in the existing `reused` ("read + from the store"), `computed` and `failed` are 0; `rows` = computed + reused + failed + + not_stored; `RunSummary.load_only: bool`. Repr line: + `rowwise: 6 rows: 3 loaded, 3 not stored (skipped)` and + `summed: 2 rows: 2 loaded (inputs incomplete: rowwise)`. Alternative: a separate `loaded` field. + Summary nodes stay function nodes only, as today. +- (f) Hashes: none move (no identity, path or record-id change). A load-only run writes nothing: + tests snapshot every table path and row count of the store before and after, both + orchestrators, including the operator cache in LOG and a side-effect log. + +## C. `node.run()` cascade (developer's rule) + +- Today (measured): `node.run()` never runs upstream. With an un-run upstream function node it + reads the upstream's STORED rows (6, though the source has 3 now) and processes them; over an + empty store, or behind an un-run OFF operator, it processes nothing and returns 0 rows silently. +- **(rec)** `node.run(*, load_only=False)` runs the sub-graph of the job nodes it depends on plus + itself, in dependency order, each once (diamonds), with ONE mechanism shared with `job.run`: an + internal function that takes a node DAG and runs it on the `SyncPipelineOrchestrator` + (`job.run` passes its whole DAG; `node.run` the ancestor DAG found by walking `upstreams`, + which in a job are the job nodes). Same rows, stored records, error_policy ("continue") and + per-node counts as `job.run` for that sub-graph. Sources are read, not run. +- **D7 "not necessarily trigger upstreams"** **(rec)**: an ancestor that holds a CURRENT run is + not run again; its held rows feed its successors. Current = it holds a completed run in this + process and `is_stale` is False (the existing judgement: own `last_modified` set and no upstream + or producer modified later; a re-run or rebound upstream makes it stale) and, for a full + cascade, that run was not load-only. So after a load-only fill, a full `node.run()` re-runs + every load-only-filled ancestor in full first (computes their missing entries) **(rec, + matches your expectation)**; option (b): use the loaded rows as they are. A load-only cascade + accepts a fully-run ancestor as current (its rows are all stored). +- Side-effect ancestors: a full cascade delivers exactly as `job.run` (with `track_completion` + only the undelivered); a load-only cascade delivers nothing (passes rows through). An ancestor + side-effect node that holds a current run (#377) is not run again, so nothing is re-delivered. +- Counts: `node.run()` still returns the node's `StreamView`. **(rec)** the cascade's + `RunSummary` is kept on the node as `node.last_run` (mirrors `job.last_run`), and a cascade + with failed rows issues the same `RowFailuresWarning` as `job.run`. Option: attach it to the + returned view (`view.run_summary`). +- Commits: as a node run on its own does today (every row) plus a flush at the end. Option: the + job's `commit_every_rows=100` cadence. +- **D6 conflict with a standing rule**: CLAUDE.md (ITL-1081, developer) "Every stream, source and + node has `run()` with no arguments ... Never give `run()` arguments", enforced by + `tests/test_core/streams/test_run.py::test_every_stream_class_has_run_without_arguments`. + **(rec)** amend to: "`run()` takes no positional arguments; a job node's `run()` takes one + keyword-only argument, `load_only`" and narrow the test to that. Option: `load_only` on + `job.run` only, and `node.run_load_only()`... (not recommended). + +## D. Loaded jobs (`PipelineJob.load`) + +- Every loaded node is empty before a run (function stubs and operators alike; operators no + longer raise from a read). `node.as_source()` reads everything stored (measured: loaded + `rowwise.as_source()` = 6 rows), with or without sources. +- `loaded.run(load_only=True)` **(rec)**: allowed. Sources come from the file (CSV, Delta) or + `loaded.bind(sources=...)` (in-memory sources are not saved). Nodes below an unbound source are + excluded and stay empty, as in any partial run. A function stub can do a load-only run: entry + ids need no function pod (system tags + input data hash), and the stub reads through its stored + paths and the saved run's content hash. Operators are rebuilt from the blueprint and run. + Without any live source there are no inputs to look up: a load-only run is impossible below + that source, and `as_source()` is the only way in. +- `loaded.run()` without `load_only`, and a loaded `node.run()`: raise as today, message pointing + to `run(load_only=True)` and `as_source()`. Loaded `node.run(load_only=True)`: the cascade. +- D8: confirm (rec) above. Risk: the stub's lookups use the saved run's content hash; a bound + source with another `source_id` finds nothing (all inputs counted not stored). + +## E. Other load statuses (D9) + +`CACHE_ONLY` (no upstream) and `READ_ONLY` with a proxy pod are reachable only through +`FunctionJobNode.from_descriptor(..., function_pod=...)` (tests only). Today their `run()` returns +every stored row. **(rec)** both empty before a run; `READ_ONLY` proxy: `run(load_only=True)` +loads, `run()` raises (cannot compute); `CACHE_ONLY`: no input, so `run(load_only=True)` raises +pointing to `as_source()`. (Candidates for removal later; out of scope.) + +## F. REPLAY (D1, not decided by me) + +Measured: after a run over 3 rows a REPLAY node holds the union of every LOG run (4 groups, two +from a 6-row run) and feeds it downstream. The rule forces only the BEFORE-a-run part (empty). +Options: (R1) REPLAY after a run keeps meaning "the stored output instead of computing": the +union of every logged run under its content hash; only the before-run read changes **(minimal +for the tag)**. (R2) REPLAY replays only the latest logged run: `odb_v1` stores no run id or time, +so this needs a new stored column -> layout change (`odb_v2`), not for the tag. (R3) REPLAY +replays only the logged rows of THIS run's inputs: operator outputs have no input entry ids, so +not possible without a stored-schema change either. + +## G. Tests and docs that change (e) + +Experiment (every no-run read empty, reverted): 400 of 4,464 job-related tests fail in 21 files: +test_node_head_sorting 162, test_loaded_operator_cache 74, test_function_node_arrow_types 58, +test_loaded_job_readback 29, test_delta_arrow_type_roundtrip 25, test_node_head 9, +test_loaded_logical_schemas 7, test_provisional_hash_warning 6, test_function_pod_node_stream 5, +test_stored_data_context 4, test_function_node_empty_data 4, test_job_node_rows_after_run 3 +(#378's `test_raised_run_reads_as_no_run`), test_tensor_job_load_back 2, +test_materialize_source_info 2, test_function_node_caching 2, test_cached_function_pod_job 2, +test_operator_node 1, test_operator_node_non_active 1, test_function_node_iteration 1, +test_readback_datagram_id 1, test_polars_dataframe_values 1, test_doc_samples +(function-pods.md) 1; plus test_run.py's no-argument test (D6) and the loaded-job "cannot run" +tests (test_loaded_job_still_cannot_run, ..._operator_only_job_cannot_run, +..._without_a_store_cannot_run, test_loaded_function_node_cannot_execute message). Each is +changed to read after `run()`/`run(load_only=True)`, or to `as_source()` where it is about the +stored records. Workshop scenario tests: unaffected (they run first, or read the store directly). +Full list: plan file appendix. + +Docs: docs/concepts/streams.md (run table and head table rows for "A job node", the "stream that +has all of its rows" paragraph, ~262), docs/concepts/function-pods.md (339-341 no-run read; +"Saving and loading a job" 662-746 and its sample 689-694, the only doc sample that reads a node +without a run: becomes bind + `loaded.run(load_only=True)`, or `as_source()`; a new "Loading +without computing" subsection), docs/concepts/sources.md 169-172; docstrings of +`PipelineJob.load/run/run_async`, node `head/as_table/iter_data/run/clear_cache`, operator +`iter_data/as_table/head`, `ViewStatus`, `StreamBase.head`. Changelog fragment `ITL-1095.md` +(Breaking: a node read before a run is empty, loaded nodes too, `node.run()` cascades; Added: +`load_only`, summary fields). CLAUDE.md/.zed/rules: replaces #378's sentence, amends the run() +rule (D6). + +## H. PR #346 (g) + +Only `tests/test_core/alignment/test_nodes.py::read_back(how="load")` reads a loaded job +without a run: `test_rows_read_back_from_the_store_are_aligned` (216 parametrizations: loaded +`scaled`/`joined` `as_table`/`head`) and `test_stored_records_read_back_are_aligned` (54; reads +`iter_data`). It will need `loaded.bind(sources=...)` (its sources are in-memory) and +`loaded.run(load_only=True)` before `return loaded`; `joined` is attached in REPLAY (D1 decides +what it then holds). `test_job.py` and the rerun modes read after a run: unaffected. + +## Sizing + +Large for Sunday: node rule + load-only through both orchestrators + summary + cascade + loaded +jobs + ~400 test updates + docs. If something must be cut for the tag, I would drop first D's +loaded-job load-only run (loaded reads empty, `as_source()` the way in, `loaded.run()` raises), +then C's "current ancestor" skip (a cascade always runs every ancestor). From 1d990178f715dcf6aa264f1015ab8106585c99b7 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:23:53 -0800 Subject: [PATCH 04/48] test(nodes): failing tests: a job node reads empty and stale before a run (ITL-1095) Co-Authored-By: Claude Opus 5.5 --- .../test_job_node_current_run_view.py | 208 ++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 tests/test_pipeline/test_job_node_current_run_view.py diff --git a/tests/test_pipeline/test_job_node_current_run_view.py b/tests/test_pipeline/test_job_node_current_run_view.py new file mode 100644 index 000000000..5b6cf24cd --- /dev/null +++ b/tests/test_pipeline/test_job_node_current_run_view.py @@ -0,0 +1,208 @@ +"""A job node shows the current run; ``node.as_source()`` shows what is stored (ITL-1095). + +Developer, November 13, 2026: a job node itself (``as_table``, ``iter_data``, the dataframe +methods, ``flow``, ``head``, the preview, ``run``) is the current run view; ``as_source()`` +is the view of everything stored for the node. Before a run in this process a node is empty +and stale, whatever its store holds; a run fills it. + +One store holds the results of two earlier runs (6 source rows, then 3). A fresh job over +that store, and a job loaded from a saved file, read empty from every node until they run. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.core.streams.view import ViewStatus +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.sources import DictSource +from orcapod.types import CacheMode + +CALLS: list[str] = [] + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + """Return ``x + 1``.""" + CALLS.append("inc") + return x + 1 + + +@function_pod(output_keys="total") +def total(y: list[int]) -> int: + """Return the sum of a group's values.""" + CALLS.append("total") + return sum(y) + + +ROWS = [{"g": i % 2, "k": i, "x": 10 * i} for i in range(6)] +ORCHESTRATORS = {"sync": SyncPipelineOrchestrator, "async": AsyncPipelineOrchestrator} +JOB_NODES = ("rowwise", "grouped", "summed") + + +@pytest.fixture(autouse=True) +def _reset_calls() -> None: + CALLS.clear() + + +def _build( + data: list[dict[str, int]], + db: Any, + cache_mode: CacheMode | None = None, +) -> PipelineJob: + """``rowwise`` (function) -> ``grouped`` (GroupBy operator) -> ``summed`` (function).""" + src = DictSource(data, tag_columns=["g", "k"], source_id="s") + job = PipelineJob(name="j", database=db) + with job: + rowwise = inc.pod(src, label="rowwise") + grouped = rowwise.group_by("g", label="grouped") + total.pod(grouped, label="summed") + if cache_mode is not None: + job.nodes["grouped"].attach_databases( + pipeline_database=db.at("j"), cache_mode=cache_mode + ) + return job + + +def _populated_store( + orchestrator: str = "sync", cache_mode: CacheMode | None = None +) -> InMemoryArrowDatabase: + """A store holding the results of a run over 6 rows and a run over 3 rows.""" + db = InMemoryArrowDatabase() + for data in (ROWS, ROWS[:3]): + _build(data, db, cache_mode).run(orchestrator=ORCHESTRATORS[orchestrator]()) + CALLS.clear() + return db + + +def _assert_empty(node: Any, columns: list[str]) -> None: + """Every read of ``node`` returns zero rows with the node's columns.""" + table = node.as_table() + assert table.num_rows == 0 + assert table.column_names == columns + assert list(node.iter_data()) == [] + assert node.as_polars_df().height == 0 + assert node.head().table.num_rows == 0 + assert node.head().table.column_names == columns + + +def _assert_points_to_run_and_stored_view(node: Any) -> None: + """The head of a node that has not run says so and names both ways in.""" + head = node.head() + assert head.status is ViewStatus.NOT_COMPUTED + header = head.header() + assert "not run" in header + assert "run()" in header + assert "as_source()" in header + assert "not run" in node._repr_html_() + + +class TestFreshJobOverAPopulatedStore: + """A job built over a store that holds results reads empty until it runs.""" + + @pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) + def test_every_read_of_every_node_is_empty_before_a_run( + self, orchestrator: str + ) -> None: + db = _populated_store(orchestrator) + job = _build(ROWS[:3], db) + expected_columns = { + label: _build(ROWS[:3], db).nodes[label].as_table().column_names + for label in JOB_NODES + } + for label in JOB_NODES: + node = job.nodes[label] + _assert_empty(node, expected_columns[label]) + _assert_points_to_run_and_stored_view(node) + assert CALLS == [] + + def test_the_stored_view_shows_every_stored_record(self) -> None: + db = _populated_store() + job = _build(ROWS[:3], db) + # Both earlier runs' rows: 6 rowwise results, 2 + 2 group totals. + assert job.nodes["rowwise"].as_source().as_table().num_rows == 6 + assert job.nodes["summed"].as_source().as_table().num_rows == 4 + assert job.nodes["rowwise"].as_table().num_rows == 0 + + def test_every_node_of_a_fresh_job_is_stale(self) -> None: + job = _build(ROWS[:3], _populated_store()) + for label in JOB_NODES: + assert job.nodes[label].is_stale, label + + @pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) + def test_a_run_fills_the_nodes_with_this_runs_rows(self, orchestrator: str) -> None: + db = _populated_store(orchestrator) + job = _build(ROWS[:3], db) + job.run(orchestrator=ORCHESTRATORS[orchestrator]()) + assert CALLS == [] # every result was stored + assert job.nodes["rowwise"].as_table().num_rows == 3 + assert sorted( + (r["g"], r["total"]) for r in job.nodes["summed"].as_table().to_pylist() + ) == [(0, 22), (1, 11)] + for label in JOB_NODES: + assert not job.nodes[label].is_stale, label + assert job.nodes[label].head().status is ViewStatus.ROWS, label + + +@pytest.mark.parametrize("mode", [CacheMode.OFF, CacheMode.LOG, CacheMode.REPLAY]) +def test_an_operator_node_reads_empty_before_a_run_in_every_cache_mode( + mode: CacheMode, +) -> None: + """REPLAY included: before a run it no longer shows every logged run.""" + # LOG runs fill the operator cache, so REPLAY has stored rows to (not) show. + db = _populated_store(cache_mode=CacheMode.LOG) + job = _build(ROWS[:3], db, cache_mode=mode) + node = job.nodes["grouped"] + _assert_empty(node, ["g", "y"]) + _assert_points_to_run_and_stored_view(node) + assert node.is_stale + + +class TestAfterARun: + """What empties a node that held a run: clearing it, or a change upstream.""" + + @pytest.mark.parametrize("label", list(JOB_NODES)) + def test_clear_cache_empties_the_node_and_makes_it_stale(self, label: str) -> None: + job = _build(ROWS[:3], InMemoryArrowDatabase()) + job.run() + node = job.nodes[label] + columns = node.as_table().column_names + node.clear_cache() + assert node.is_stale + _assert_empty(node, columns) + + def test_a_rebound_source_empties_its_dependent_nodes(self) -> None: + job = _build(ROWS[:3], InMemoryArrowDatabase()) + job.run() + source_name = next(iter(job.sources)) # the one source slot + job.bind( + sources={ + source_name: DictSource(ROWS, tag_columns=["g", "k"], source_id="s") + } + ) + for label in JOB_NODES: + node = job.nodes[label] + assert node.is_stale, label + assert node.as_table().num_rows == 0, label + assert node.head().status is ViewStatus.NOT_COMPUTED, label + + +def test_a_loaded_job_reads_empty_until_it_runs(tmp_path: Path) -> None: + """A loaded job's nodes are empty and do not raise; ``as_source()`` reads the store.""" + db = InMemoryArrowDatabase() + job = _build(ROWS, db) + job.run() + job.save(tmp_path / "job.json") + loaded = PipelineJob.load(tmp_path / "job.json", database=db) + for label in JOB_NODES: + node = loaded.nodes[label] + assert node.as_table().num_rows == 0, label + assert list(node.iter_data()) == [], label + assert node.head().status is ViewStatus.NOT_COMPUTED, label + assert "not run" in node.head().header(), label + assert loaded.nodes["rowwise"].as_source().as_table().num_rows == 6 From b1a8d2174865286621b480755c1c836aeb356841 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:27:02 -0800 Subject: [PATCH 05/48] feat(nodes)!: a job node reads empty and stale before a run (ITL-1095) A function or operator job node shows only the rows of its current run in this process: before a run, after clear_cache(), after a run that raised, and once stale (any job node it depends on, or a rebound source, changed since) every read returns zero typed rows and head() points to run() and node.as_source(). REPLAY no longer shows stored rows before a run; a loaded node reads empty instead of every stored row (function) or raising (operator). Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/function_node.py | 188 ++++++------------ src/orcapod/core/nodes/operator_node.py | 165 +++++++-------- src/orcapod/core/nodes/source_node.py | 27 ++- src/orcapod/core/nodes/staleness.py | 34 ++++ src/orcapod/core/streams/view.py | 7 +- .../test_job_node_rows_after_run.py | 23 +-- 6 files changed, 207 insertions(+), 237 deletions(-) create mode 100644 src/orcapod/core/nodes/staleness.py diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index aa9c50dff..7b28f3b4d 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -36,10 +36,13 @@ ) from orcapod.core.streams.view import ( BLUEPRINT_NOTE, + JOB_NODE_NOT_RUN_NOTE, + STALE_NOTE, ViewStatus, StreamView, ) from orcapod.core.tracker import DEFAULT_TRACKER_MANAGER +from orcapod.core.nodes.staleness import upstream_job_node_stale from orcapod.core.datagrams.tag_data import Data, EmptyData, Tag from orcapod.errors import ( EphemeralResultMissingError, @@ -1117,6 +1120,8 @@ def __init__( result_database=result_database, ephemeral_database=ephemeral_database, ) + # A node that has not run in this process holds nothing and is stale (ITL-1095). + self._set_modified_time(None) # ------------------------------------------------------------------ # node_config property @@ -1513,14 +1518,27 @@ def from_descriptor( def clear_cache(self) -> None: """Clear in-memory output caches, content hash cache, and node identity path cache. - The node then holds no run: it reads as a node with no completed run in this - process (every row stored under its content hash). + The node then holds no run: it is stale and reads empty, as before its first run + in this process (ITL-1095). Its stored results stay stored (``as_source()``). """ super().clear_cache() self._cached_output_datas.clear() self._cached_output_table = None self._cached_content_hash_column = None self._run_entry_counts = None + # No content established: the node, and every node downstream, is stale. + self._set_modified_time(None) + + @property + def is_stale(self) -> bool: + """Whether the rows this node holds are out of date, or it holds none. + + True before the node's first run in this process and after ``clear_cache()`` + (no modified time), when an input or the function pod was modified after the node's + run (``StreamBase.is_stale``), or when any job node it depends on is stale + (ITL-1095). A stale node reads empty until it runs again. + """ + return super().is_stale or upstream_job_node_stale(self.upstreams) def _held_pairs(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: """Yield the (tag, data) rows this node holds; every read of them goes through here. @@ -1529,17 +1547,18 @@ def _held_pairs(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: (two equal inputs give two rows, as downstream received them). The in-memory cache may hold more (results of other inputs, read from the store), which stay cached for reuse but are not rows of this run (ITL-1093). Without a completed - run in this process, every held row. Rows come in the order the cache holds - them. An input without output (an inactive function) is not a row. + run in this process there are no rows, whatever the cache or the store holds + (ITL-1095). Rows come in the order the cache holds them. An input without output + (an inactive function) is not a row. """ - held = self._cached_output_datas counts = self._run_entry_counts if counts is None: - pairs: Iterable[tuple[TagProtocol, DataProtocol | None]] = held.values() - else: - pairs = ( - pair for eid, pair in held.items() for _ in range(counts.get(eid, 0)) - ) + return + pairs: Iterable[tuple[TagProtocol, DataProtocol | None]] = ( + pair + for eid, pair in self._cached_output_datas.items() + for _ in range(counts.get(eid, 0)) + ) for tag, data in pairs: if data is not None: yield tag, data @@ -3136,18 +3155,15 @@ def head( columns: ColumnConfig | dict[str, Any] | None = None, all_info: bool = False, ) -> StreamView: - """Return the first rows the node holds or has stored; never runs the function. + """Return the first rows of the node's current run; never runs the function. After a run (or ``execute``) the rows are the node's results in memory for that - run's inputs, the first of what ``as_table`` returns. Otherwise they are the first of its stored - rows: the node's whole stored slice is read and nothing is kept on the node. - With nothing stored, there are no rows. The node has all of its rows, so with - ``sort_by_tags`` (or ``all_info``) the head is exactly - ``as_table(...).slice(0, n)``, whatever order the rows were computed in. See - ``StreamBase.head``. - - Warns: - StoreContentWarning: As for ``as_table``. + run's inputs, the first of what ``as_table`` returns. A node that has not run in + this process, or whose input changed since (stale), shows no rows and says so, + pointing to ``run()`` and to ``as_source()`` for its stored results (ITL-1095); + the store is not read. The node has all of its rows, so with ``sort_by_tags`` + (or ``all_info``) the head is exactly ``as_table(...).slice(0, n)``, whatever + order the rows were computed in. See ``StreamBase.head``. """ from orcapod.pipeline.serialization import LoadStatus @@ -3158,51 +3174,15 @@ def head( return self._head_without_rows( config, ViewStatus.UNAVAILABLE, note="no store attached" ) - current = status == LoadStatus.CACHE_ONLY or not self.is_stale - if current and ( - self._run_entry_counts is not None or self._cached_output_datas - ): - return self._head_of_pairs( - list(self._held_pairs()), n, config, origin="held" + if self._run_entry_counts is None: + return self._head_without_rows( + config, ViewStatus.NOT_COMPUTED, note=JOB_NODE_NOT_RUN_NOTE ) - return self._head_from_store(n, config) - - def _head_from_store(self, n: int, config: ColumnConfig) -> StreamView: - """Return the first ``n`` of this node's stored rows. - - Reads the node's whole stored slice (pipeline table joined with the result - table; no bounded read exists yet) and keeps nothing on the node. - """ - if self._function_pod is None: - # A loaded node: its table is built from the stored Arrow records. - self._check_expected_rows() - fetched = self._fetch_joined_records() - stored = None if fetched is None else self._stored_rows_stream(fetched) - stored_ids = set(stored[0]) if stored is not None else set() - missing = 0 - if fetched is not None: - missing = sum( - 1 for k in fetched.empty_data_tokens if k not in stored_ids - ) - note = ( - f"{missing:,} {'result' if missing == 1 else 'results'} not shown: kept " - "in an ephemeral store, which is not saved with the job" - if missing - else None + if status != LoadStatus.CACHE_ONLY and self.is_stale: + return self._head_without_rows( + config, ViewStatus.NOT_COMPUTED, note=STALE_NOTE ) - if stored is not None: - return ( - stored[1] - .head(n, columns=config) - .for_stream(self, origin="stored", note=note) - ) - return self._head_without_rows(config, ViewStatus.NOTHING_STORED, note=note) - if self._cached_function_pod is None: - return self._head_without_rows(config, ViewStatus.NOT_COMPUTED) - entries = list(self._load_cached_entries().values()) - if not any(data is not None for _, data in entries): - return self._head_without_rows(config, ViewStatus.NOTHING_STORED) - return self._head_of_pairs(entries, n, config, origin="stored") + return self._head_of_pairs(list(self._held_pairs()), n, config, origin="held") # ------------------------------------------------------------------ # Iteration @@ -3214,54 +3194,33 @@ def as_table( columns: ColumnConfig | dict[str, Any] | None = None, all_info: bool = False, ) -> "pa.Table": - """Return this node's results as a PyArrow table. + """Return the rows of this node's current run as a PyArrow table. - A node loaded by ``PipelineJob.load`` has no function pod. Its table is - built from the Arrow records its saved run stored, so every column keeps - its stored type. Row order follows the store. With nothing stored, the - table has zero rows and the node's columns. + Read-only: never computes and never reads the store. Before a run in this + process, or once the node is stale, the table has zero rows and the node's + columns (ITL-1095); ``as_source()`` reads what the store holds. Raises: - RuntimeError: If the node was loaded and no store is attached, or some of - its results were kept in an ephemeral store, which is not saved with - the job. - - Warns: - StoreContentWarning: Once per store, if the node was loaded from a job file - that records another number of pipeline records for its run than the - store holds. The read returns what the store holds. + RuntimeError: If the node was loaded and no store is attached. """ - if self._function_pod is None: - self._require_readable_stub() - fetched = self._fetch_joined_records() - if fetched is None: - return super().as_table(columns=columns, all_info=all_info) - stored = self._stored_rows_stream(fetched) - stored_ids = set(stored[0]) if stored is not None else set() - ephemeral = [k for k in fetched.empty_data_tokens if k not in stored_ids] - if ephemeral: - raise RuntimeError( - f"FunctionJobNode {self.label!r}: the saved run kept " - f"{len(ephemeral)} of its results in an ephemeral store, which " - "is not saved with the job, so they cannot be read into a " - "table. iter_data() yields them as EmptyData placeholders." - ) - if stored is not None: - return stored[1].as_table(columns=columns, all_info=all_info) + from orcapod.pipeline.serialization import LoadStatus + + if self.load_status == LoadStatus.UNAVAILABLE: + raise self._no_store_error() return super().as_table(columns=columns, all_info=all_info) def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: - """Yield this node's (tag, data) rows. + """Yield the (tag, data) rows of this node's current run. - Strictly read-only — never triggers computation. Callers must call - ``run()`` or ``execute()`` first if they want results computed. + Strictly read-only: never computes and never reads the store. Callers must + call ``run()`` (or run the job) first. After a completed run (``execute``, ``async_execute``, ``run()`` or a job run) the rows are exactly those of that run's inputs, on either orchestrator, even - though results of other inputs stay cached and stored (ITL-1093). A run that - raised defines no rows. Without a completed run in this process, on the first - call with an empty in-memory store and a DB attached, hot-loads every record - stored under the node's content hash (one-shot, no recompute) and yields them. + though results of other inputs stay cached and stored (ITL-1093). A node that + has not run in this process, whose run raised, or that is stale (its input + changed since its run) yields nothing (ITL-1095); ``as_source()`` reads every + row stored for it. Raises: RuntimeError: If ``load_status`` is UNAVAILABLE. @@ -3271,34 +3230,9 @@ def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: status = self.load_status if status == LoadStatus.UNAVAILABLE: raise self._no_store_error() - - if status == LoadStatus.CACHE_ONLY: - # Upstream unavailable; serve entirely from DB. - if not self._cached_output_datas: - loaded = self._load_cached_entries() - self._cached_output_datas.update(loaded) - if loaded: - self._cached_output_table = None - self._cached_content_hash_column = None - yield from self._held_pairs() - return - - # FULL / READ_ONLY — in-memory store may be populated from computation - # (via execute/run) or hot-loaded from DB. - if self.is_stale: + # A CACHE_ONLY node has no live input to be stale against. + if status != LoadStatus.CACHE_ONLY and self.is_stale: self.clear_cache() - - if self._function_pod is None: - # A loaded node: its rows can come only from the store. - self._check_expected_rows() - if not self._cached_output_datas and self._cached_function_pod is not None: - # Hot-load from DB on the first call when store is empty. - loaded = self._load_cached_entries() - self._cached_output_datas.update(loaded) - if loaded: - self._cached_output_table = None - self._cached_content_hash_column = None - yield from self._held_pairs() def run(self) -> StreamView: diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index bf2517c92..edb9e0b73 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -32,6 +32,7 @@ check_head_size, ) from orcapod.core.streams.view import ( + JOB_NODE_NOT_RUN_NOTE, STALE_NOTE, BLUEPRINT_NOTE, ViewStatus, @@ -39,6 +40,7 @@ head_of, ) from orcapod.core.tracker import DEFAULT_TRACKER_MANAGER +from orcapod.core.nodes.staleness import upstream_job_node_stale from orcapod.errors import ( PipelineJobRequiredError, StoreReadError, @@ -710,6 +712,8 @@ def __init__( pipeline_database=pipeline_database, cache_mode=cache_mode, ) + # A node that has not run in this process holds nothing and is stale (ITL-1095). + self._set_modified_time(None) # ------------------------------------------------------------------ # as_node — return the lightweight OperatorNode equivalent @@ -739,10 +743,16 @@ def set_ephemeral_store(self, store: "ArrowDatabaseProtocol | None") -> None: # ------------------------------------------------------------------ def clear_cache(self) -> None: - """Clear output caches and node identity path cache.""" + """Clear output caches and node identity path cache. + + The node then holds no run: it is stale and reads empty, as before its first run + in this process (ITL-1095). Rows it stored stay stored (``as_source()``). + """ super().clear_cache() # clears _node_identity_path_cache + _update_modified_time self._cached_output_stream = None self._cached_output_table = None + # No content established: the node, and every node downstream, is stale. + self._set_modified_time(None) # ------------------------------------------------------------------ # attach_databases @@ -1424,7 +1434,10 @@ def run(self) -> StreamView: ) self._update_modified_time() computed = True - table = self.as_table() + # Read the kept output itself: a lazy input that finishes computing during this + # read stamps itself after the node, which would make as_table() judge it stale. + assert self._cached_output_stream is not None + table = self._cached_output_stream.as_table() if computed: # Without a database and in OFF mode the output stream is lazy: as_table() # computes it, and an uncomputed input finishes only then. Stamp the node @@ -1437,26 +1450,41 @@ def run(self) -> StreamView: origin = "computed" if kept is None else head_of(kept.head(1), self).origin return self._view_of_rows(table, origin) - def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: - """Return an iterator over (tag, data) pairs. + @property + def is_stale(self) -> bool: + """Whether the rows this node holds are out of date, or it holds none. + + True before the node's first run in this process and after ``clear_cache()`` + (no modified time), when an input or the operator was modified after the node's + run (``StreamBase.is_stale``), or when any job node it depends on is stale + (ITL-1095). A stale node reads empty until it runs again. + """ + return super().is_stale or upstream_job_node_stale(self.upstreams) - Read-only: never triggers computation. Yields the rows of the latest - ``run()``, ``execute()`` or ``async_execute()``. With nothing cached in - memory (before a run, or after ``clear_cache()``), a node in - ``CacheMode.REPLAY`` with a database yields the rows stored there; in - any other mode it yields nothing. Call ``node.is_stale`` before - iterating if you need to detect outdated cached data. + def _current_output(self) -> StreamProtocol | None: + """The output of the node's current run, or ``None`` when it holds none. - Loaded nodes raise when no readable stored cache is available. + A node holds a run's output from the end of ``run()``, ``execute()`` or + ``async_execute()`` until it is cleared or becomes stale (an input or the + operator changed since). Before a run in this process it holds none, whatever + its cache mode and store hold (ITL-1095). """ - if self._cached_output_stream is not None: - return self._cached_output_stream.iter_data() - db_stream = self._load_cached_stream_from_db() - if db_stream is not None: - return db_stream.iter_data() - if self._loaded: - raise self._no_stored_results_error() - return iter([]) + if self._cached_output_stream is None or self.is_stale: + return None + return self._cached_output_stream + + def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: + """Return an iterator over the (tag, data) rows of the node's current run. + + Read-only: never computes and never reads the store. Yields the rows of the + latest ``run()``, ``execute()`` or ``async_execute()``; nothing before a run in + this process, after ``clear_cache()`` or once the node is stale, in every cache + mode (ITL-1095). ``as_source()`` reads what the node stored (``LOG``). + """ + current = self._current_output() + if current is None: + return iter([]) + return current.iter_data() def as_table( self, @@ -1464,34 +1492,24 @@ def as_table( columns: ColumnConfig | dict[str, Any] | None = None, all_info: bool = False, ) -> "pa.Table": - """Return the output as a PyArrow table. + """Return the output of the node's current run as a PyArrow table. - Read-only: never triggers computation. Returns the rows of the latest - ``run()``, ``execute()`` or ``async_execute()``. With nothing cached in - memory (before a run, or after ``clear_cache()``), a node in - ``CacheMode.REPLAY`` with a database returns the rows stored there; in - any other mode it returns a zero-row table with the node's columns. - - Loaded nodes raise when no readable stored cache is available. + Read-only: never computes and never reads the store. Returns the rows of the + latest ``run()``, ``execute()`` or ``async_execute()``; before a run in this + process, after ``clear_cache()`` or once the node is stale, a zero-row table + with the node's columns, in every cache mode (ITL-1095). """ - if self._cached_output_stream is not None: - return self._cached_output_stream.as_table( - columns=columns, all_info=all_info - ) - db_stream = self._load_cached_stream_from_db() - if db_stream is not None: - return db_stream.as_table(columns=columns, all_info=all_info) - if self._loaded: - raise self._no_stored_results_error() - # No cached or stored records yet: construct a zero-row table with the - # correct schema and route it through ArrowTableStream so that the - # ColumnConfig / all_info shaping is applied consistently. + current = self._current_output() + if current is not None: + return current.as_table(columns=columns, all_info=all_info) # Requires a live operator for schema derivation (pre-existing limitation). if self._operator is None: return pa.table({}) - empty_records = self._make_empty_table() - tag_keys = self.keys()[0] - empty_stream = ArrowTableStream(empty_records, tag_columns=tag_keys) + empty_stream = ArrowTableStream( + self._make_empty_table(), + tag_columns=self.keys()[0], + data_context=self.data_context, + ) return empty_stream.as_table(columns=columns, all_info=all_info) def head( @@ -1501,66 +1519,25 @@ def head( columns: ColumnConfig | dict[str, Any] | None = None, all_info: bool = False, ) -> StreamView: - """Return the first rows the node holds or has stored; never runs the operator. + """Return the first rows of the node's current run; never runs the operator. - After a run (or ``execute``) the rows are the node's output in memory. Before - that, a node in ``CacheMode.REPLAY`` shows the first of its stored rows, read - from its whole stored table; in ``LOG`` and ``OFF`` it shows none, as - ``as_table`` does. A loaded node never raises here. See ``StreamBase.head``. + After a run (or ``execute``) the rows are the node's output in memory. Before a + run in this process the node shows no rows and says so, pointing to ``run()`` + and to ``as_source()``, in every cache mode; once stale it says its inputs + changed (ITL-1095). The store is never read. See ``StreamBase.head``. """ check_head_size(n) config = ColumnConfig.handle_config(columns, all_info=all_info) - if self._cached_output_stream is not None: - if not self._loaded and self.is_stale: - # run() would recompute; the kept rows are not the current result. - return self._head_without_rows( - config, ViewStatus.NOT_COMPUTED, note=STALE_NOTE - ) - return head_of(self._cached_output_stream.head(n, columns=config), self) - if self._cache_mode == CacheMode.REPLAY and self._pipeline_database is not None: - return self._head_from_store(n, config) - if self._loaded: - return self._head_without_rows( - config, - ViewStatus.UNAVAILABLE, - note=( - "loaded node whose results are not stored (cache mode OFF)" - if self._cache_mode == CacheMode.OFF - else "loaded node whose stored results are not replayed " - f"(cache mode {self._cache_mode.value.upper()})" - ), - ) - if self._cache_mode == CacheMode.LOG and self._pipeline_database is not None: - note = ( - "not run in this session; rows of earlier runs are stored but not " - "replayed (cache mode LOG)" - ) - else: - note = "not run; cache mode OFF stores nothing" - return self._head_without_rows(config, ViewStatus.NOT_COMPUTED, note=note) - - def _head_from_store(self, n: int, config: ColumnConfig) -> StreamView: - """Return the first ``n`` of this node's stored rows. - - Reads the node's whole stored slice (no bounded read exists yet) through the - same path as ``as_table`` and keeps nothing on the node. A loaded node whose - run's rows cannot be read (its cache table is absent, or none of its rows is the - run's) shows no rows and says why, where ``as_table`` raises. - """ - try: - stream = self._load_cached_stream_from_db() - except _LoadedCacheUnreadableError as exc: + if self._cached_output_stream is None: return self._head_without_rows( - config, ViewStatus.UNAVAILABLE, note=exc.head_note + config, ViewStatus.NOT_COMPUTED, note=JOB_NODE_NOT_RUN_NOTE ) - if stream is None or stream.as_table().num_rows == 0: + if self.is_stale: + # run() would recompute; the kept rows are not the current result. return self._head_without_rows( - config, - ViewStatus.NOTHING_STORED, - # A loaded job cannot run: say what the store holds instead. - note="the store holds no rows for its run" if self._loaded else None, + config, ViewStatus.NOT_COMPUTED, note=STALE_NOTE ) - return stream.head(n, columns=config).for_stream(self, origin="stored") + return head_of(self._cached_output_stream.head(n, columns=config), self) # ------------------------------------------------------------------ # DB retrieval diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 8d7cf8923..a7535d2b1 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -10,6 +10,7 @@ import functools import logging from abc import ABC, abstractmethod +from datetime import datetime from collections.abc import AsyncIterator, Iterator from typing import TYPE_CHECKING, Any @@ -597,10 +598,34 @@ def bound_source(self) -> StreamProtocol | None: @bound_source.setter def bound_source(self, value: StreamProtocol | None) -> None: - """Bind *value* as the concrete source and invalidate both hash caches.""" + """Bind *value* as the concrete source and invalidate both hash caches. + + Binding stamps the slot as modified, so every job node fed by it is stale and + reads empty until it runs again (ITL-1095). + """ self._bound_source = value self._invalidate_content_hash_cache() self._invalidate_pipeline_hash_cache() + self._update_modified_time() + + @property + def last_modified(self) -> datetime | None: + """When the slot's content last changed: its binding or its bound source. + + The later of the time the slot was bound and the bound source's own + ``last_modified`` (a ``CachedSource`` or ``PollingSource`` moves it when it + reads new rows). A bound source that does not know its modification time + leaves the binding time. A source whose rows change without its modified time + moving (a table read lazily, a file rewritten in place) is not seen: build a + new source object and bind it, or clear the dependent nodes. + """ + own = self._modified_time + bound = ( + self._bound_source.last_modified if self._bound_source is not None else None + ) + if own is None or bound is None: + return own if bound is None else bound + return max(own, bound) # ------------------------------------------------------------------ # Identity — delegate to bound source when set diff --git a/src/orcapod/core/nodes/staleness.py b/src/orcapod/core/nodes/staleness.py new file mode 100644 index 000000000..b5320cce4 --- /dev/null +++ b/src/orcapod/core/nodes/staleness.py @@ -0,0 +1,34 @@ +"""Staleness of job nodes across the job graph (ITL-1095). + +A stream is stale when an upstream or its producer was modified after it +(``StreamBase.is_stale``), which looks one level up only. A job node holds the rows of +its current run, so it is stale also when any job node it depends on is stale: a source +rebound two nodes up makes every node below it stale, not only its direct consumer. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + + +def upstream_job_node_stale(upstreams: Iterable[Any]) -> bool: + """Return whether any function or operator job node in ``upstreams`` is stale. + + Each such node answers with its own ``is_stale``, which applies this rule to its + own upstreams in turn, so the whole ancestry is covered. Side-effect job nodes are + not followed yet: their own ``is_stale`` is redefined by ITL-1091. + + Args: + upstreams: The direct upstream streams of a job node. + + Returns: + ``True`` if any upstream function or operator job node is stale. + """ + from orcapod.core.nodes.function_node import FunctionJobNode + from orcapod.core.nodes.operator_node import OperatorJobNode + + return any( + isinstance(upstream, (FunctionJobNode, OperatorJobNode)) and upstream.is_stale + for upstream in upstreams + ) diff --git a/src/orcapod/core/streams/view.py b/src/orcapod/core/streams/view.py index 8417e984f..f440bde36 100644 --- a/src/orcapod/core/streams/view.py +++ b/src/orcapod/core/streams/view.py @@ -46,7 +46,8 @@ class ViewStatus(str, Enum): means the content is empty. PARTIAL: A computation stopped part way; the rows are those computed so far. NOT_COMPUTED: Nothing is computed, or what was computed is out of date. - NOTHING_STORED: A job node whose store holds no rows for it yet. + NOTHING_STORED: A stored view (``node.as_source()``) whose store holds no rows + for its node yet. UNAVAILABLE: No rows can exist here: a pipeline blueprint, an input slot with no source bound, or a source that is not available. """ @@ -63,6 +64,10 @@ class ViewStatus(str, Enum): "not computed: inputs changed since the last computation; run() recomputes it" ) NOTHING_STORED_NOTE = "nothing stored; run the job" +# A job node shows the rows of its current run only; before one it shows none (ITL-1095). +JOB_NODE_NOT_RUN_NOTE = ( + "not run; run() fills it; node.as_source() shows every stored result" +) BLUEPRINT_NOTE = "pipeline blueprint; run it in a PipelineJob" SIDE_EFFECTS_NOTE = ( "not run; run() delivers its side effects; displaying never delivers side effects" diff --git a/tests/test_pipeline/test_job_node_rows_after_run.py b/tests/test_pipeline/test_job_node_rows_after_run.py index fd57128b0..d859e68cd 100644 --- a/tests/test_pipeline/test_job_node_rows_after_run.py +++ b/tests/test_pipeline/test_job_node_rows_after_run.py @@ -5,8 +5,8 @@ the rows of that run's inputs through ``as_table``, ``iter_data``, ``head`` and ``run()``, and so does a consumer built over the node afterwards; the two orchestrators agree row for row. Stored results of other inputs stay stored and reusable. A run that raised defines no -rows: the node then reads as a node with no run in this process (every row stored under -its content hash). +rows: the node then reads as a node with no run in this process, which is empty (ITL-1095); +what was stored stays readable through ``as_source()``. """ from __future__ import annotations @@ -275,13 +275,7 @@ def add_one(x: int) -> int: class TestRaisedRun: - """A run that raised defines no rows: the node reads as a node with no run.""" - - @staticmethod - def _no_run_read(fn: Any, data: list[dict[str, int]], db: Any) -> list[Any]: - fresh = _single(fn.pod, data, db) - fresh.compile() - return _values(fresh.nodes["node"], lambda r: (r["k"], r["y"])) + """A run that raised defines no rows: the node reads as a node with no run (empty).""" @pytest.mark.parametrize( ("kind", "fn", "policy"), @@ -299,12 +293,13 @@ def test_raised_run_reads_as_no_run(self, kind: str, fn: Any, policy: str) -> No with pytest.raises((ValueError, StoreWriteError)): job.run(orchestrator=_orchestrator(kind, policy)) node = job.nodes["node"] - expected = self._no_run_read(fn, ROWS[:4], db) - assert _values(node, lambda r: (r["k"], r["y"])) == expected + assert _values(node, lambda r: (r["k"], r["y"])) == [] reads = _reads(node) - assert reads["head"] == reads["as_table"] - # Nothing stored was dropped: the stored rows of the earlier run are among them. - assert {(3, 31), (4, 41), (5, 51)} <= set(expected) + assert reads["head"] == reads["as_table"] == [] + assert node.is_stale + # Nothing stored was dropped: the stored view holds the earlier run's rows. + stored = {(r["k"], r["y"]) for r in node.as_source().as_table().to_pylist()} + assert {(3, 31), (4, 41), (5, 51)} <= stored class TestDuplicateInputs: From 39aa109406573cce755cb673aa6b7d0094820073 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:33:58 -0800 Subject: [PATCH 06/48] test: read job nodes after a run, not before (ITL-1095) Co-Authored-By: Claude Opus 5.5 --- .../function_pod/test_ephemeral_result.py | 15 ++--- .../test_function_node_arrow_types.py | 7 ++- .../test_function_node_caching.py | 2 + .../test_function_pod_node_stream.py | 33 ++++++++-- .../nodes/test_function_node_iteration.py | 7 ++- tests/test_pipeline/test_node_head.py | 62 ++++++++++++------- tests/test_pipeline/test_node_head_sorting.py | 13 +++- .../test_node_staleness_after_run.py | 6 +- 8 files changed, 98 insertions(+), 47 deletions(-) diff --git a/tests/test_core/function_pod/test_ephemeral_result.py b/tests/test_core/function_pod/test_ephemeral_result.py index b7fb8dbeb..53bec7e79 100644 --- a/tests/test_core/function_pod/test_ephemeral_result.py +++ b/tests/test_core/function_pod/test_ephemeral_result.py @@ -581,8 +581,8 @@ def test_iter_data_serves_result_after_cross_session_recompute(self): iter_data() serves the fresh result without triggering additional computation. Verifies that: - - after clearing the in-memory cache, iter_data() hot-loads the recomputed - result from the pipeline + ephemeral DBs (exercises _load_cached_entries) + - after clearing the node, its next run loads the recomputed result from the + pipeline + ephemeral DBs (exercises _load_cached_entries) - repeated iter_data() calls return the same result without recomputing """ call_count = {"n": 0} @@ -621,11 +621,12 @@ def counting_double(x: int) -> int: node2.execute(stream) assert call_count["n"] == 2 # recomputed once due to cross-session miss - # execute() calls _update_modified_time() at the end, so is_stale is False here. - # Manually clearing forces the iter_data() hot-load branch (_load_cached_entries → - # _fetch_joined_records) without relying on staleness, so this tests the DB join - # path explicitly rather than the in-memory dict fast-path. - node2._cached_output_datas.clear() + # Clearing the node drops its in-memory results; running it again loads the + # recomputed result through the DB join path (_load_cached_entries → + # _fetch_joined_records) rather than the in-memory dict fast-path. A node shows + # rows only after a run (ITL-1095). + node2.clear_cache() + node2.execute(stream) # iter_data() must serve the recomputed result — no additional computation results = list(node2.iter_data()) diff --git a/tests/test_core/function_pod/test_function_node_arrow_types.py b/tests/test_core/function_pod/test_function_node_arrow_types.py index 0a3764a2a..e2253d91f 100644 --- a/tests/test_core/function_pod/test_function_node_arrow_types.py +++ b/tests/test_core/function_pod/test_function_node_arrow_types.py @@ -230,7 +230,7 @@ def test_as_table_mixing_stored_and_new_rows_keeps_tag_type() -> None: def test_store_read_back_keeps_tag_type( name: str, store: str, tmp_path: pathlib.Path ) -> None: - """Reads from the store (``get_all_records``, a fresh job's reads) keep the tag type.""" + """Reads from the store (``get_all_records``, a fresh job's run) keep the tag type.""" database = _new_store(store, tmp_path) table = _source_table(name) _build(database, table).run() @@ -238,11 +238,14 @@ def test_store_read_back_keeps_tag_type( # A new store object sees the committed tables, as a new process would. database = _new_store(store, tmp_path) - fresh = _build(database, table).nodes["double"] + fresh_job = _build(database, table) + fresh = fresh_job.nodes["double"] records = fresh.get_all_records() assert records is not None assert records.schema.field("k").type == _expected(name) + # The node shows rows once its run has read them from the store (ITL-1095). + fresh_job.run() tag, _ = next(iter(fresh.iter_data())) assert tag.as_table().schema.field("k").type == _expected(name) assert fresh.as_table().schema.field("k").type == _expected(name) diff --git a/tests/test_core/function_pod/test_function_node_caching.py b/tests/test_core/function_pod/test_function_node_caching.py index c58ba463b..2e194db9b 100644 --- a/tests/test_core/function_pod/test_function_node_caching.py +++ b/tests/test_core/function_pod/test_function_node_caching.py @@ -231,6 +231,7 @@ def test_phase1_yields_existing_results(self): # Create a second node with the same stream and shared DB stream2 = _make_stream([{"id": 0, "x": 10}, {"id": 1, "x": 20}]) node2, _ = _make_node(stream2, db=db) + node2.run() # a node shows its rows only after a run (ITL-1095) results = list(node2.iter_data()) # All results should come from Phase 1 (DB), not recomputed @@ -319,6 +320,7 @@ def test_all_existing_entry_ids_skipped_in_phase2(self): node2._versioned_pipeline_path ).num_rows + node2.run() results = list(node2.iter_data()) assert len(results) == 2 diff --git a/tests/test_core/function_pod/test_function_pod_node_stream.py b/tests/test_core/function_pod/test_function_pod_node_stream.py index 2ab30e0e7..bda5c9e06 100644 --- a/tests/test_core/function_pod/test_function_pod_node_stream.py +++ b/tests/test_core/function_pod/test_function_pod_node_stream.py @@ -169,7 +169,8 @@ def test_as_table_returns_cached_results_when_data_function_inactive( ): """ Cache filled by node1 (active) is shared with node2 (inactive). - node2.as_table() must return full results from Phase 1 (DB). + node2's run serves the full results from Phase 1 (DB); a node shows its rows + only after a run (ITL-1095). """ n = 3 db = InMemoryArrowDatabase() @@ -181,6 +182,7 @@ def test_as_table_returns_cached_results_when_data_function_inactive( double_pf.set_active(False) node2 = _make_node(double_pf, n=n, db=db) + node2.run() table2 = node2.as_table() assert isinstance(table2, pa.Table) @@ -201,6 +203,7 @@ def test_inactive_filled_db_serves_cached_results(self, double_pf): double_pf.set_active(False) node2 = _make_node(double_pf, n=n, db=db) + node2.run() # a node shows its rows only after a run (ITL-1095) data = list(node2.iter_data()) assert len(data) == n @@ -266,7 +269,9 @@ def test_db_served_results_have_correct_row_count(self, double_pf): n = 5 db = InMemoryArrowDatabase() _fill_node(_make_node(double_pf, n=n, db=db)) - data = list(_make_node(double_pf, n=n, db=db).iter_data()) + node2 = _make_node(double_pf, n=n, db=db) + node2.run() # a node shows its rows only after a run (ITL-1095) + data = list(node2.iter_data()) assert len(data) == n def test_fresh_db_always_computes(self, double_pf): @@ -373,9 +378,11 @@ def counting_double(x: int) -> int: class TestFunctionNodeStaleness: # --- is_stale --- - def test_is_stale_false_immediately_after_creation(self, double_pf): - """A freshly created FunctionNode whose upstream has not changed is not stale.""" + def test_is_stale_immediately_after_creation(self, double_pf): + """A freshly created job node holds no run, so it is stale (ITL-1095).""" node = _make_node(double_pf, n=3) + assert node.is_stale + node.run() assert not node.is_stale def test_is_stale_true_after_upstream_modified(self, double_pf): @@ -395,7 +402,7 @@ def test_is_stale_true_after_upstream_modified(self, double_pf): assert node.is_stale - def test_is_stale_false_after_clear_cache(self, double_pf): + def test_is_stale_after_clear_cache_until_it_runs(self, double_pf): import time db = InMemoryArrowDatabase() @@ -405,13 +412,16 @@ def test_is_stale_false_after_clear_cache(self, double_pf): input_stream=input_stream, pipeline_database=db, ) - list(node.iter_data()) + node.run() time.sleep(0.01) input_stream._update_modified_time() assert node.is_stale + # A cleared node holds no run: it stays stale until it runs (ITL-1095). node.clear_cache() + assert node.is_stale + node.run() assert not node.is_stale # --- clear_cache --- @@ -431,6 +441,9 @@ def test_clear_cache_produces_same_results_on_re_iteration(self, double_pf): table_before = node.as_table() node.clear_cache() + # A cleared node reads empty until it runs again (ITL-1095). + assert node.as_table().num_rows == 0 + node.run() table_after = node.as_table() assert sorted(table_before.column("result").to_pylist()) == sorted( @@ -449,12 +462,17 @@ def test_iter_data_auto_detects_stale_and_repopulates(self, double_pf): input_stream=input_stream, pipeline_database=db, ) + node.run() first = list(node.iter_data()) + assert len(first) == 3 time.sleep(0.01) input_stream._update_modified_time() assert node.is_stale + # A stale node reads empty (ITL-1095); its run reads the stored results again. + assert list(node.iter_data()) == [] + node.run() second = list(node.iter_data()) assert len(second) == len(first) assert [p["result"] for _, p in second] == [p["result"] for _, p in first] @@ -476,6 +494,9 @@ def test_as_table_auto_detects_stale_and_repopulates(self, double_pf): time.sleep(0.01) input_stream._update_modified_time() + # A stale node reads empty (ITL-1095); its run reads the stored results again. + assert node.as_table().num_rows == 0 + node.run() table_after = node.as_table() assert len(table_after) == 3 assert sorted(table_after.column("result").to_pylist()) == sorted( diff --git a/tests/test_core/nodes/test_function_node_iteration.py b/tests/test_core/nodes/test_function_node_iteration.py index fee923c42..e28d54b4d 100644 --- a/tests/test_core/nodes/test_function_node_iteration.py +++ b/tests/test_core/nodes/test_function_node_iteration.py @@ -59,14 +59,17 @@ def test_iter_does_not_call_process_data_internal(self): list(node.iter_data()) mock_proc.assert_not_called() - def test_iter_after_db_populated_hot_loads_without_compute(self): - """iter_data() on a node with DB records hot-loads without _process_data_internal.""" + def test_iter_after_db_populated_yields_nothing_until_a_run_loads(self): + """A node over a populated DB yields nothing before a run (ITL-1095); its run + loads every result from the DB without _process_data_internal.""" db = InMemoryArrowDatabase() node1 = _make_node(n=3, db=db) node1.run() # populate DB node2 = _make_node(n=3, db=db) + assert list(node2.iter_data()) == [] with patch.object(node2, "_process_data_internal") as mock_proc: + node2.run() results = list(node2.iter_data()) mock_proc.assert_not_called() assert len(results) == 3 diff --git a/tests/test_pipeline/test_node_head.py b/tests/test_pipeline/test_node_head.py index 7cbea6a69..e4c821561 100644 --- a/tests/test_pipeline/test_node_head.py +++ b/tests/test_pipeline/test_node_head.py @@ -98,10 +98,12 @@ def test_nodes_before_a_run_show_no_rows_and_compute_nothing() -> None: op_html = job.nodes["sel"]._repr_html_() assert "could not read" not in op_html assert CALLS == [] - assert fn_head.status is ViewStatus.NOTHING_STORED + # A node that has not run says so and names both ways in (ITL-1095). + for head in (fn_head, op_head): + assert head.status is ViewStatus.NOT_COMPUTED + assert "not run" in head.header() + assert "as_source()" in head.header() assert fn_head.table.column_names == ["id", "y"] - assert op_head.status is ViewStatus.NOT_COMPUTED - assert "cache mode OFF" in op_head.header() @pytest.mark.parametrize("store_kind", ["in_memory", "delta"]) @@ -129,26 +131,33 @@ def test_nodes_after_a_run_show_their_first_rows( @pytest.mark.parametrize("store_kind", ["in_memory", "delta"]) -def test_a_new_job_over_a_filled_store_reads_stored_rows_without_computing( +def test_a_new_job_over_a_filled_store_shows_rows_once_its_run_reads_the_store( store_kind: str, tmp_path: Path ) -> None: - """A new job over a filled store shows stored rows without computing.""" + """A new job over a filled store shows no rows until it runs (ITL-1095); its run + reads every result from the store and computes nothing.""" store = _store(store_kind, tmp_path) _ = _job(store).run() CALLS.clear() - node = _job(store).nodes["triple"] + job = _job(store) + node = job.nodes["triple"] + assert node.head(4).status is ViewStatus.NOT_COMPUTED + # The stored view shows the stored rows meanwhile. + assert node.as_source().head(4).status is ViewStatus.ROWS + _ = job.run() head = node.head(4) assert CALLS == [] assert head.status is ViewStatus.ROWS - # The whole stored slice is read, so the header states the exact count and - # makes no claim of a bounded read. - assert head.header() == "FunctionJobNode[triple] — first 4 of 25 stored rows" + assert head.header() == "FunctionJobNode[triple] — first 4 of 25 rows" assert head.table.equals(node.as_table().slice(0, 4)) -def test_a_loaded_job_shows_stored_rows_and_never_raises(tmp_path: Path) -> None: - """A loaded job's nodes show stored rows, or say why there are none.""" +def test_a_loaded_job_shows_no_rows_until_it_runs_and_never_raises( + tmp_path: Path, +) -> None: + """A loaded job's nodes show no rows before a run (ITL-1095) and say where the + stored rows are; ``as_source()`` shows them.""" store = DeltaTableDatabase(base_path=tmp_path / "store") job = _job(store) _ = job.run() @@ -160,10 +169,12 @@ def test_a_loaded_job_shows_stored_rows_and_never_raises(tmp_path: Path) -> None fn_head = loaded.nodes["triple"].head() op_head = loaded.nodes["sel"].head() assert CALLS == [] - assert fn_head.status is ViewStatus.ROWS - assert fn_head.header().endswith("first 10 of 25 stored rows") - assert op_head.status is ViewStatus.UNAVAILABLE - assert "cache mode OFF" in op_head.header() + for head in (fn_head, op_head): + assert head.status is ViewStatus.NOT_COMPUTED + assert "as_source()" in head.header() + stored = loaded.nodes["triple"].as_source().head() + assert stored.status is ViewStatus.ROWS + assert stored.header().endswith("first 10 of 25 stored rows") def test_blueprint_nodes_are_unavailable() -> None: @@ -207,27 +218,27 @@ def _operator_node( def test_operator_node_cache_modes() -> None: - """Operator nodes show stored rows in REPLAY and none in LOG before a run.""" + """Before a run an operator node shows no rows in every cache mode (ITL-1095); a + REPLAY node shows the stored rows once it has run.""" store = InMemoryArrowDatabase() assert _operator_node(CacheMode.REPLAY, store).head().status is ( - ViewStatus.NOTHING_STORED + ViewStatus.NOT_COMPUTED ) seed = _operator_node(CacheMode.LOG, store) _ = seed.execute(_source()) replay = _operator_node(CacheMode.REPLAY, store) + assert replay.head(3).status is ViewStatus.NOT_COMPUTED + _ = replay.run() head = replay.head(3) assert head.status is ViewStatus.ROWS - assert ( - head.header() - == "OperatorJobNode[SelectDataColumns] — first 3 of 25 stored rows" - ) + assert head.header() == "OperatorJobNode[SelectDataColumns] — first 3 of 25 rows" assert head.table.equals(replay.as_table().slice(0, 3)) log = _operator_node(CacheMode.LOG, store) log_head = log.head() assert log_head.status is ViewStatus.NOT_COMPUTED - assert "not replayed" in log_head.header() + assert "as_source()" in log_head.header() @pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) @@ -268,7 +279,7 @@ def test_head_keeps_the_tag_arrow_type_of_the_input( """A node's head keeps an ``int8`` tag ``int8`` and is the prefix of ``as_table``. ``head`` and ``as_table`` share one table build (ITL-928 typing), in memory after a - run and from the store in a new job. + run and in a new job whose run reads its rows from the store. """ table = pa.table( {"id": pa.array(range(12), pa.int8()), "x": pa.array(range(12), pa.int64())}, @@ -293,7 +304,10 @@ def build(store: ArrowDatabaseProtocol) -> PipelineJob: store = _store(store_kind, tmp_path) job = build(store) _ = job.run(orchestrator=ORCHESTRATORS[orchestrator]()) - for node in (job.nodes["triple"], build(store).nodes["triple"]): + # The new job shows rows once its run has read them from the store (ITL-1095). + fresh = build(store) + _ = fresh.run(orchestrator=ORCHESTRATORS[orchestrator]()) + for node in (job.nodes["triple"], fresh.nodes["triple"]): head = node.head(5, all_info=True) assert head.status is ViewStatus.ROWS assert head.table.schema.field("id").type == pa.int8() diff --git a/tests/test_pipeline/test_node_head_sorting.py b/tests/test_pipeline/test_node_head_sorting.py index 42dccf48f..bc4e38793 100644 --- a/tests/test_pipeline/test_node_head_sorting.py +++ b/tests/test_pipeline/test_node_head_sorting.py @@ -121,7 +121,8 @@ def _held_order(node: Any) -> list[int]: def test_a_function_nodes_head_is_the_prefix_of_as_table_in_any_completion_order( run_kind: str, store_kind: str, n: int, config: str, tmp_path: Path ) -> None: - """After a run, and from the store in a new job, head(n, cfg) == as_table(cfg)[:n].""" + """After a run, and in a new job whose run reads every row from the store, + head(n, cfg) == as_table(cfg)[:n].""" store = _store(store_kind, tmp_path) job = _run(run_kind, store) assert _held_order(job.nodes["node"]) != sorted(_held_order(job.nodes["node"])) @@ -131,6 +132,9 @@ def test_a_function_nodes_head_is_the_prefix_of_as_table_in_any_completion_order store, list(range(N_ROWS)), ) + # A node shows its current run only (ITL-1095): the new job's run reads its rows from + # the store, in the store's order, and computes nothing. + fresh.run() for node in (job.nodes["node"], fresh.nodes["node"]): head = node.head(n, **CONFIGS[config]) assert head.status is ViewStatus.ROWS @@ -175,11 +179,14 @@ def node(mode: CacheMode) -> OperatorJobNode: ran = node(CacheMode.LOG) ran.execute(_source(order)) - for each in (ran, node(CacheMode.REPLAY)): + # A REPLAY node shows the stored rows once it has run (ITL-1095). + replayed = node(CacheMode.REPLAY) + replayed.run() + for each in (ran, replayed): head = each.head(3, **CONFIGS[config]) assert head.status is ViewStatus.ROWS assert head.table.equals(each.as_table(**CONFIGS[config]).slice(0, 3)) - sorted_ids = node(CacheMode.REPLAY).head(3, all_info=True).table.column("id") + sorted_ids = replayed.head(3, all_info=True).table.column("id") assert sorted_ids.to_pylist() == [0, 1, 2] diff --git a/tests/test_pipeline/test_node_staleness_after_run.py b/tests/test_pipeline/test_node_staleness_after_run.py index fae00b2dc..9c0e6a23a 100644 --- a/tests/test_pipeline/test_node_staleness_after_run.py +++ b/tests/test_pipeline/test_node_staleness_after_run.py @@ -325,8 +325,8 @@ def test_a_function_node_is_stamped_before_its_output_closes(with_store: bool) - pipeline_database=store, result_database=store, ) - constructed = node.last_modified - assert constructed is not None + # A node that has not run holds nothing and has no modified time (ITL-1095). + assert node.last_modified is None seen_at_close: list[Any] = [] async def run() -> None: @@ -353,4 +353,4 @@ async def read() -> None: asyncio.run(run()) (stamp,) = seen_at_close - assert stamp is not None and stamp > constructed + assert stamp is not None From f1c36f22a3b8160b5dc0d3a1c03650df82efa193 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:35:05 -0800 Subject: [PATCH 07/48] test: read stored results of loaded jobs through as_source() (ITL-1095) Co-Authored-By: Claude Opus 5.5 --- .../test_cached_function_pod_job.py | 5 ++-- .../test_polars_dataframe_values.py | 12 ++++++--- .../function_pod/test_readback_datagram_id.py | 5 +++- .../nodes/test_function_node_empty_data.py | 26 ++++++++++++++----- .../test_provisional_hash_warning.py | 3 ++- tests/test_torch/test_tensor_job_load_back.py | 3 ++- 6 files changed, 39 insertions(+), 15 deletions(-) diff --git a/tests/test_core/function_pod/test_cached_function_pod_job.py b/tests/test_core/function_pod/test_cached_function_pod_job.py index 6ada9c6d4..e14541b98 100644 --- a/tests/test_core/function_pod/test_cached_function_pod_job.py +++ b/tests/test_core/function_pod/test_cached_function_pod_job.py @@ -233,9 +233,10 @@ def test_saved_job_equals_the_uncached_pods_and_loads( doc["bindings"].pop("database") assert cached_saved == plain_saved - # A loaded job reads its stored results; the pod cache is not part of it. + # A loaded job's stored results are read through the stored view (its nodes are + # empty until it runs, ITL-1095); the pod cache is not part of it. loaded = PipelineJob.load(tmp_path / "cached.json", database=store) - assert loaded.nodes["add"].as_table().to_pylist() == [ + assert loaded.nodes["add"].as_source().as_table().to_pylist() == [ {"k": 1, "total": 5}, {"k": 2, "total": 10}, ] diff --git a/tests/test_core/function_pod/test_polars_dataframe_values.py b/tests/test_core/function_pod/test_polars_dataframe_values.py index 170d0f53b..c18a38515 100644 --- a/tests/test_core/function_pod/test_polars_dataframe_values.py +++ b/tests/test_core/function_pod/test_polars_dataframe_values.py @@ -462,19 +462,23 @@ def test_saved_job_loaded_back_reads_the_frames(self, tmp_path: Path) -> None: node = loaded.nodes["make"] _, data_schema = node.output_schema() assert data_schema["df"] is pl.DataFrame - got = {tag["k"]: data["df"] for tag, data in node.iter_data()} + # A loaded job's nodes are empty until it runs (ITL-1095); its stored results + # are read through the stored view. + stored_view = node.as_source() + got = {tag["k"]: data["df"] for tag, data in stored_view.iter_data()} _assert_exact(got["a"], _frame(1)) _assert_exact(got["b"], _frame(3)) stored = dict( zip( - node.as_table().column("k").to_pylist(), - _stored_frames(node.as_table().column("df")), + stored_view.as_table().column("k").to_pylist(), + _stored_frames(stored_view.as_table().column("df")), strict=True, ) ) _assert_exact(stored["a"], _frame(1)) heights = { - tag["k"]: data["height"] for tag, data in loaded.nodes["height"].iter_data() + tag["k"]: data["height"] + for tag, data in loaded.nodes["height"].as_source().iter_data() } assert heights == {"a": 1, "b": 3} diff --git a/tests/test_core/function_pod/test_readback_datagram_id.py b/tests/test_core/function_pod/test_readback_datagram_id.py index 661df3598..9b922b8ae 100644 --- a/tests/test_core/function_pod/test_readback_datagram_id.py +++ b/tests/test_core/function_pod/test_readback_datagram_id.py @@ -89,7 +89,10 @@ def test_a_fresh_job_on_the_store_reads_the_stored_ids() -> None: _ = first.run(orchestrator=SyncPipelineOrchestrator()) stored = stored_ids(first) - fresh = build(store) # never run: every row comes from the store + fresh = build(store) + # A node shows rows only after a run (ITL-1095): this run computes nothing, every + # row comes from the store. + _ = fresh.run(orchestrator=SyncPipelineOrchestrator()) assert read_back_ids(fresh) == stored diff --git a/tests/test_core/nodes/test_function_node_empty_data.py b/tests/test_core/nodes/test_function_node_empty_data.py index 3158dc148..7edf9222d 100644 --- a/tests/test_core/nodes/test_function_node_empty_data.py +++ b/tests/test_core/nodes/test_function_node_empty_data.py @@ -174,6 +174,20 @@ def test_normal_result_wins_over_empty_data_token(self, persistent_node): assert isinstance(data, Data) +def _entries(node: FunctionJobNode) -> list: + """The node's entries for its input rows, as its cache façade returns them. + + These tests drive ``execute_data`` (one row, not a run), so the node shows no rows + of a run (ITL-1095); ``get_cached_results`` gives the entries it holds or has + stored, an ``EmptyData`` placeholder for a result whose ephemeral store is gone. + """ + entry_ids = [ + node.compute_base_entry_id(tag, data) + for tag, data in node._input_stream.iter_data() + ] + return list(node.get_cached_results(entry_ids).values()) + + class TestProcessDataInternalEmptyData: def test_cache_hit_returns_cached_result(self, ephemeral_node): """Downstream cache hit on EmptyData input returns the cached result.""" @@ -203,7 +217,7 @@ def triple_value(result: int) -> int: ) # Step 3: Run downstream with real data to populate its result cache - for tag, data in node.iter_data(): + for tag, data in _entries(node): downstream.execute_data(tag, data) # Step 4: Simulate upstream ephemeral expiry @@ -211,7 +225,7 @@ def triple_value(result: int) -> int: node._cached_output_datas.clear() # Step 5: Upstream now emits EmptyData - upstream_items = list(node.iter_data()) + upstream_items = _entries(node) assert len(upstream_items) == 1 tag_in, data_in = upstream_items[0] assert isinstance(data_in, EmptyData) @@ -252,7 +266,7 @@ def triple_value(result: int) -> int: result_database=InMemoryArrowDatabase(), ) - upstream_items = list(node.iter_data()) + upstream_items = _entries(node) tag_in, data_in = upstream_items[0] assert isinstance(data_in, EmptyData) @@ -296,7 +310,7 @@ def triple_value(result: int) -> int: ) # Populate downstream result cache with real data (sync) - for tag, data in node.iter_data(): + for tag, data in _entries(node): downstream.execute_data(tag, data) # Simulate ephemeral expiry @@ -304,7 +318,7 @@ def triple_value(result: int) -> int: node._cached_output_datas.clear() # Upstream now emits EmptyData - upstream_items = list(node.iter_data()) + upstream_items = _entries(node) assert len(upstream_items) == 1 tag_in, data_in = upstream_items[0] assert isinstance(data_in, EmptyData) @@ -341,7 +355,7 @@ def triple_value(result: int) -> int: result_database=InMemoryArrowDatabase(), ) - upstream_items = list(node.iter_data()) + upstream_items = _entries(node) tag_in, data_in = upstream_items[0] assert isinstance(data_in, EmptyData) diff --git a/tests/test_logical_types/test_provisional_hash_warning.py b/tests/test_logical_types/test_provisional_hash_warning.py index 5a450ee09..c3e102a73 100644 --- a/tests/test_logical_types/test_provisional_hash_warning.py +++ b/tests/test_logical_types/test_provisional_hash_warning.py @@ -232,7 +232,8 @@ def run_and_save() -> None: _quiet(run_and_save) with recorded() as caught: loaded = PipelineJob.load(tmp_path / "job.json") - table = loaded.nodes["make"].as_table() + # A loaded node is empty until it runs (ITL-1095): read the stored view. + table = loaded.nodes["make"].as_source().as_table() assert table.num_rows == 3 assert _warned_names(caught) == [name] diff --git a/tests/test_torch/test_tensor_job_load_back.py b/tests/test_torch/test_tensor_job_load_back.py index 20b819ecd..2d1022c5c 100644 --- a/tests/test_torch/test_tensor_job_load_back.py +++ b/tests/test_torch/test_tensor_job_load_back.py @@ -85,7 +85,8 @@ def test_a_loaded_job_reads_its_stored_tensors( assert loaded.nodes["value"].output_schema()[1]["value"] is torch.Tensor assert loaded.nodes["joined"].load_status == LoadStatus.FULL assert loaded.nodes["joined"].output_schema()[1]["value"] is torch.Tensor - got = _values(loaded.nodes["value"]) + # A loaded node is empty until it runs (ITL-1095): read the stored view. + got = _values(loaded.nodes["value"].as_source()) assert sorted(got) == [1, 2] for row_id, n in ((1, 3), (2, 4)): expected = function(n) From bea357f404010bd2e326e2fcdda120751e2f312c Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:36:45 -0800 Subject: [PATCH 08/48] fix(function_node): count a run input only when it gave a row; docstrings for the new tests (ITL-1093) Addresses the CodeRabbit review of PR #378: of two equal inputs where one fails and the other succeeds, the node now holds one row, as downstream received. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/function_node.py | 13 ++- .../test_job_node_rows_after_run.py | 92 ++++++++++++++++--- 2 files changed, 89 insertions(+), 16 deletions(-) diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index aa9c50dff..252c98c6e 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -1972,6 +1972,9 @@ def execute( # and prevents spurious recomputation of already-processed data. self.get_cached_results(base_entry_ids=base_entry_ids) + # Base entry ids of the inputs that gave a row (a cache hit or a + # successful computation), one per input: the node's rows afterwards. + emitted_entry_ids: list[bytes] = [] for tag, data, base_entry_id in upstream_entries: ctx_obs.on_data_start(node_label, tag, data) @@ -1980,6 +1983,7 @@ def execute( ): tag_out, result = self._cached_output_datas[base_entry_id] ctx_obs.on_data_end(node_label, tag, data, result, cached=True) + emitted_entry_ids.append(base_entry_id) if result is not None: output.append((tag_out, result)) continue @@ -2027,6 +2031,7 @@ def execute( continue ctx_obs.on_data_end(node_label, tag, data, result, cached=False) + emitted_entry_ids.append(base_entry_id) if result is not None: output.append((tag_out, result)) self._commit_or_end_node( @@ -2046,7 +2051,7 @@ def execute( raise # The node's rows are those of this run's inputs, whatever else it caches. - self._set_run_rows(base_entry_ids) + self._set_run_rows(emitted_entry_ids) ctx_obs.on_node_end(node_label, node_hash) # Mark this node as freshly computed so subsequent iter_data() calls # skip the is_stale check and serve results directly from the in-memory cache. @@ -3460,7 +3465,8 @@ async def async_execute( # Local dict: correlation_key → (original_tag, original_input_data) input_store: dict[bytes, tuple[TagProtocol, DataProtocol, bytes]] = {} - # Base entry ids of this run's inputs, in arrival order (ITL-1093). + # Base entry ids of this run's inputs that gave a row (a cache hit or a + # forwarded computed row), one per input (ITL-1093). run_entry_ids: list[bytes] = [] async def route_inputs() -> None: @@ -3468,7 +3474,6 @@ async def route_inputs() -> None: try: async for tag, data in input_channel: base_entry_id = self.compute_base_entry_id(tag, data) - run_entry_ids.append(base_entry_id) # A cached EmptyData is a result that is gone (an # ephemeral result from an earlier run): compute the row # again, as the sync path does (ITL-923). @@ -3485,6 +3490,7 @@ async def route_inputs() -> None: ctx_obs.on_data_end( node_label, tag, data, cached_data, cached=True ) + run_entry_ids.append(base_entry_id) await output.send((cached_tag, cached_data)) else: correlation_key = uuid.uuid4().bytes @@ -3542,6 +3548,7 @@ async def record_and_forward() -> None: # Commits every commit_every_rows rows inside a job (ITL-919); # no await in between, so no other row interleaves. self._row_processed() + run_entry_ids.append(base_entry_id) await output.send((clean_tag, output_data)) # Capture outer self so _NodeLabelObserver methods can diff --git a/tests/test_pipeline/test_job_node_rows_after_run.py b/tests/test_pipeline/test_job_node_rows_after_run.py index fd57128b0..e1a53ee7e 100644 --- a/tests/test_pipeline/test_job_node_rows_after_run.py +++ b/tests/test_pipeline/test_job_node_rows_after_run.py @@ -76,10 +76,12 @@ def store_fails_on_20(x: int) -> int: def _source(data: list[dict[str, int]]) -> DictSource: + """Return a ``DictSource`` of ``data`` with tags ``g`` and ``k`` and source id ``s``.""" return DictSource(data, tag_columns=["g", "k"], source_id="s") def _build(data: list[dict[str, int]], db: Any) -> PipelineJob: + """Return a job over ``db``: ``rowwise``, ``downstream`` of it, and ``grouped``.""" job = PipelineJob(name="itl1093", database=db) with job: rowwise = inc.pod(_source(data), label="rowwise") @@ -89,6 +91,7 @@ def _build(data: list[dict[str, int]], db: Any) -> PipelineJob: def _orchestrator(kind: str, error_policy: str = "continue") -> Any: + """Return the ``sync`` or ``async`` orchestrator with ``error_policy``.""" if kind == "sync": return SyncPipelineOrchestrator(error_policy=error_policy) # pyright: ignore[reportArgumentType] return AsyncPipelineOrchestrator(error_policy=error_policy) # pyright: ignore[reportArgumentType] @@ -121,18 +124,22 @@ def _reads(node: Any) -> dict[str, list[tuple[tuple[str, str], ...]]]: def _values(node: Any, key: Callable[[dict[str, Any]], Any]) -> list[Any]: + """Return ``key`` of every row of ``node.as_table()``, sorted.""" return sorted(key(row) for row in node.as_table().to_pylist()) def _expected_rowwise(data: list[dict[str, int]]) -> list[tuple[int, int]]: + """Return the ``(k, y)`` rows ``rowwise`` gives for ``data``, sorted.""" return sorted((row["k"], row["x"] + 1) for row in data) def _expected_downstream(data: list[dict[str, int]]) -> list[tuple[int, int]]: + """Return the ``(k, z)`` rows ``downstream`` gives for ``data``, sorted.""" return sorted((row["k"], 2 * (row["x"] + 1)) for row in data) def _expected_grouped(data: list[dict[str, int]]) -> list[tuple[int, int]]: + """Return the ``(g, total)`` rows ``grouped`` gives for ``data``, sorted.""" sums: dict[int, int] = {} for row in data: sums[row["g"]] = sums.get(row["g"], 0) + row["x"] @@ -163,6 +170,7 @@ def _check_run(job: PipelineJob, data: list[dict[str, int]]) -> None: def _snapshot(job: PipelineJob) -> dict[str, Any]: + """Return every public read of each function node of ``job``.""" return { label: _reads(job.nodes[label]) for label in ("rowwise", "downstream", "grouped") @@ -174,6 +182,7 @@ class TestFreshJobPerRun: @pytest.mark.parametrize("kind", ORCHESTRATORS) def test_each_run_returns_its_inputs_rows(self, kind: str) -> None: + """Every read after each run gives that run's rows; the full run reuses 3.""" db = InMemoryArrowDatabase() for index, data in enumerate(SEQUENCE): job = _build(data, db) @@ -185,6 +194,7 @@ def test_each_run_returns_its_inputs_rows(self, kind: str) -> None: assert summary.nodes["rowwise"].computed == 3 def test_sync_and_async_agree_row_for_row(self) -> None: + """Both orchestrators give equal rows, system tags included, after every run.""" snapshots: dict[str, list[dict[str, Any]]] = {} for kind in ORCHESTRATORS: db = InMemoryArrowDatabase() @@ -197,6 +207,7 @@ def test_sync_and_async_agree_row_for_row(self) -> None: @pytest.mark.parametrize("kind", ORCHESTRATORS) def test_node_run_returns_its_inputs_rows(self, kind: str) -> None: + """A node's ``run()`` after the job run shows that run's rows.""" db = InMemoryArrowDatabase() for data in SEQUENCE: job = _build(data, db) @@ -212,6 +223,7 @@ class TestSameJobRebound: @pytest.mark.parametrize("kind", ORCHESTRATORS) def test_each_run_returns_its_inputs_rows(self, kind: str) -> None: + """Every read after each run of the rebound job gives that run's rows.""" db = InMemoryArrowDatabase() job = _build(SEQUENCE[0], db) for data in SEQUENCE: @@ -220,6 +232,7 @@ def test_each_run_returns_its_inputs_rows(self, kind: str) -> None: _check_run(job, data) def test_sync_and_async_agree_row_for_row(self) -> None: + """Both orchestrators give equal rows after every run of the rebound job.""" snapshots: dict[str, list[dict[str, Any]]] = {} for kind in ORCHESTRATORS: db = InMemoryArrowDatabase() @@ -233,6 +246,7 @@ def test_sync_and_async_agree_row_for_row(self) -> None: def _single(pod: Any, data: list[dict[str, int]], db: Any) -> PipelineJob: + """Return a job over ``db`` applying ``pod`` to a source of ``data`` as ``node``.""" job = PipelineJob(name="itl1093_single", database=db) with job: pod(_source(data), label="node") @@ -244,6 +258,7 @@ class TestNullAndInactive: @pytest.mark.parametrize("kind", ORCHESTRATORS) def test_null_value_is_a_row(self, kind: str) -> None: + """A row whose output value is null is one of the run's rows.""" db = InMemoryArrowDatabase() _single(null_on_10.pod, ROWS, db).run(orchestrator=_orchestrator(kind)) job = _single(null_on_10.pod, ROWS[:3], db) @@ -256,8 +271,11 @@ def test_null_value_is_a_row(self, kind: str) -> None: @pytest.mark.parametrize("kind", ORCHESTRATORS) def test_inactive_function_gives_no_rows(self, kind: str) -> None: + """A run of an inactive function has no rows, though other results are stored.""" + @function_pod(output_keys="y") def add_one(x: int) -> int: + """Return ``x + 1``.""" return x + 1 db = InMemoryArrowDatabase() @@ -279,6 +297,7 @@ class TestRaisedRun: @staticmethod def _no_run_read(fn: Any, data: list[dict[str, int]], db: Any) -> list[Any]: + """Return the ``(k, y)`` rows a compiled, unrun job over ``db`` reads.""" fresh = _single(fn.pod, data, db) fresh.compile() return _values(fresh.nodes["node"], lambda r: (r["k"], r["y"])) @@ -292,6 +311,7 @@ def _no_run_read(fn: Any, data: list[dict[str, int]], db: Any) -> list[Any]: ], ) def test_raised_run_reads_as_no_run(self, kind: str, fn: Any, policy: str) -> None: + """After a run that raised, the node reads every stored row, as with no run.""" db = InMemoryArrowDatabase() # Store the results of rows 3..5 first, then run rows 0..3, which fails at x=20. _single(fn.pod, ROWS[3:], db).run(orchestrator=_orchestrator(kind)) @@ -315,7 +335,8 @@ class TestDuplicateInputs: """ @staticmethod - def _node() -> Any: + def _node(pod: Any) -> Any: + """Return a node applying ``pod`` to rows (k=0, x=5) twice and (k=2, x=7).""" import pyarrow as pa from orcapod.core.nodes import FunctionJobNode @@ -332,27 +353,36 @@ def _node() -> Any: schema=schema, ) return FunctionJobNode( - inc.pod, # pyright: ignore[reportArgumentType] # the decorator's pod type is not declared as FunctionPodProtocol + pod, ArrowTableStream(table, tag_columns=["k"]), pipeline_database=InMemoryArrowDatabase(), result_database=InMemoryArrowDatabase(), ) - def test_sync(self) -> None: - node = self._node() - view = node.run() - held = sorted((r["k"], r["y"]) for r in node.as_table().to_pylist()) - assert held == [(0, 6), (0, 6), (2, 8)] - assert sorted((r["k"], r["y"]) for r in view.as_table().to_pylist()) == held + @staticmethod + def _flaky_pod() -> Any: + """Return a pod computing ``x + 1`` whose first call for ``x == 5`` raises.""" + calls: list[int] = [] - def test_async(self) -> None: + @function_pod(output_keys="y") + def flaky_inc(x: int) -> int: + """Return ``x + 1``; raise on the first call for ``x == 5``.""" + calls.append(x) + if x == 5 and calls.count(5) == 1: + raise ValueError("transient failure") + return x + 1 + + return flaky_inc.pod + + @staticmethod + def _run_async(node: Any) -> list[tuple[int, int]]: + """Run ``node`` with ``async_execute`` over its upstream; return the rows sent.""" import asyncio from orcapod.channels import Channel - node = self._node() - async def run() -> list[tuple[int, int]]: + """Feed the upstream's rows in and collect what the node sends out.""" inputs = Channel(buffer_size=16) outputs = Channel(buffer_size=16) # The input stream is the node's own upstream (a test of async_execute). @@ -366,6 +396,42 @@ async def run() -> list[tuple[int, int]]: ] return sorted(got) - received = asyncio.run(run()) + return asyncio.run(run()) + + @staticmethod + def _held(node: Any) -> list[tuple[int, int]]: + """Return the node's ``(k, y)`` rows, sorted.""" + return sorted((r["k"], r["y"]) for r in node.as_table().to_pylist()) + + def test_sync(self) -> None: + """Sync: both equal inputs are held, as the run returned them.""" + node = self._node(inc.pod) + view = node.run() + assert self._held(node) == [(0, 6), (0, 6), (2, 8)] + assert sorted((r["k"], r["y"]) for r in view.as_table().to_pylist()) == ( + self._held(node) + ) + + def test_async(self) -> None: + """Async: both equal inputs are held, as they were sent downstream.""" + node = self._node(inc.pod) + received = self._run_async(node) assert received == [(0, 6), (0, 6), (2, 8)] - assert sorted((r["k"], r["y"]) for r in node.as_table().to_pylist()) == received + assert self._held(node) == received + + def test_sync_one_of_two_equal_inputs_fails(self) -> None: + """Sync: an equal input that failed is not held; the one that succeeded is.""" + node = self._node(self._flaky_pod()) + received = sorted( + (tag.as_dict()["k"], data.as_dict()["y"]) + for tag, data in node.execute(node.upstreams[0]) + ) + assert received == [(0, 6), (2, 8)] + assert self._held(node) == received + + def test_async_one_of_two_equal_inputs_fails(self) -> None: + """Async: an equal input that failed is not held; the one that succeeded is.""" + node = self._node(self._flaky_pod()) + received = self._run_async(node) + assert received == [(0, 6), (2, 8)] + assert self._held(node) == received From 55e01eb2cd57781bb84cde2d12d3fd4222e738d7 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:36:58 -0800 Subject: [PATCH 09/48] test: update operator, delta and stored-context reads to the current-run rule (ITL-1095) Co-Authored-By: Claude Opus 5.5 --- .../operators/test_materialize_source_info.py | 2 ++ .../test_core/operators/test_operator_node.py | 3 +++ .../test_operator_node_non_active.py | 9 ++++++-- .../test_delta_arrow_type_roundtrip.py | 21 +++++++++++++------ 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/tests/test_core/operators/test_materialize_source_info.py b/tests/test_core/operators/test_materialize_source_info.py index 59a7c9f8b..057291375 100644 --- a/tests/test_core/operators/test_materialize_source_info.py +++ b/tests/test_core/operators/test_materialize_source_info.py @@ -694,6 +694,7 @@ def test_rerun_after_pre_fix_store(self, monkeypatch: pytest.MonkeyPatch): ) replay = self._node(db, CacheMode.REPLAY) + _ = replay.run() # a node shows rows only after a run (ITL-1095) assert len(list(replay.iter_data())) == 5 def test_v014_store_is_not_read(self, tmp_path: Path): @@ -739,6 +740,7 @@ def test_v014_store_is_not_read(self, tmp_path: Path): tokens = _source_info_tokens(list(_pin_source().iter_data())) assert _stored_source(new) == tokens replay = self._node(db, CacheMode.REPLAY) + _ = replay.run() # a node shows rows only after a run (ITL-1095) assert _source_info_tokens(list(replay.iter_data())) == tokens diff --git a/tests/test_core/operators/test_operator_node.py b/tests/test_core/operators/test_operator_node.py index 1c54d45c6..53cab260f 100644 --- a/tests/test_core/operators/test_operator_node.py +++ b/tests/test_core/operators/test_operator_node.py @@ -375,6 +375,9 @@ def test_replay_from_cache(self, simple_stream, db): node_replay = _make_node( op, (simple_stream,), db=db, cache_mode=CacheMode.REPLAY ) + # A node shows rows only after a run (ITL-1095); REPLAY's run reads the cache. + assert node_replay.as_table().num_rows == 0 + node_replay.run() table = node_replay.as_table() assert table.num_rows == 3 assert "renamed_x" in table.column_names diff --git a/tests/test_core/operators/test_operator_node_non_active.py b/tests/test_core/operators/test_operator_node_non_active.py index 52922a3de..c3215141f 100644 --- a/tests/test_core/operators/test_operator_node_non_active.py +++ b/tests/test_core/operators/test_operator_node_non_active.py @@ -160,8 +160,11 @@ def test_iter_data_replay_mode_no_records_returns_empty( result = list(node.iter_data()) assert result == [] - def test_iter_data_replay_mode_returns_db_contents(self, simple_source, map_op): - """REPLAY + prior LOG run → iter_data returns DB records without run().""" + def test_iter_data_replay_mode_returns_db_contents_after_its_run( + self, simple_source, map_op + ): + """REPLAY + prior LOG run → iter_data returns nothing before the REPLAY node's + run (ITL-1095) and the DB records after it.""" db = InMemoryArrowDatabase() # Populate DB via LOG run node_log = _node(map_op, (simple_source,), db=db, cache_mode=CacheMode.LOG) @@ -171,6 +174,8 @@ def test_iter_data_replay_mode_returns_db_contents(self, simple_source, map_op): node_replay = _node( map_op, (simple_source,), db=db, cache_mode=CacheMode.REPLAY ) + assert list(node_replay.iter_data()) == [] + node_replay.run() result = list(node_replay.iter_data()) assert len(result) == 3 for _, data in result: diff --git a/tests/test_databases/test_delta_arrow_type_roundtrip.py b/tests/test_databases/test_delta_arrow_type_roundtrip.py index f6ba81599..d87183636 100644 --- a/tests/test_databases/test_delta_arrow_type_roundtrip.py +++ b/tests/test_databases/test_delta_arrow_type_roundtrip.py @@ -194,7 +194,9 @@ def _chained_job(root: pathlib.Path, table: pa.Table) -> PipelineJob: def test_a_rerun_on_the_same_store_writes_no_pipeline_rows( name: str, mode: str, tmp_path: pathlib.Path ) -> None: - """The ITL-990 case: source -> pod -> pod, run twice; a fresh job reads 3 rows.""" + """The ITL-990 case: source -> pod -> pod, run twice; a fresh job's run shows 3 rows + and the store holds 3 records per node (a node shows rows only after a run, + ITL-1095).""" from orcapod.pipeline import AsyncPipelineOrchestrator table = _source_table(TAGS[name]) @@ -206,6 +208,7 @@ def test_a_rerun_on_the_same_store_writes_no_pipeline_rows( job.run() fresh = _chained_job(tmp_path, table) + fresh.run() for label in ("double", "increment"): node = fresh.nodes[label] assert node.as_table().num_rows == 3 @@ -217,11 +220,14 @@ def test_a_rerun_on_the_same_store_writes_no_pipeline_rows( @pytest.mark.parametrize("name", list(TAGS)) def test_a_fresh_job_reads_the_tag_type(name: str, tmp_path: pathlib.Path) -> None: - """A new job on the same store reads the tag type through every read of a node.""" + """A new job on the same store, run, reads the tag type through every read of a + node (a node shows rows only after a run, ITL-1095).""" table = _source_table(TAGS[name]) _chained_job(tmp_path, table).run() - fresh = _chained_job(tmp_path, table).nodes["double"] + fresh_job = _chained_job(tmp_path, table) + fresh_job.run() + fresh = fresh_job.nodes["double"] assert fresh.as_table().schema.field("k").type == TAGS[name].type tag, _ = next(iter(fresh.iter_data())) @@ -232,7 +238,8 @@ def test_a_fresh_job_reads_the_tag_type(name: str, tmp_path: pathlib.Path) -> No # Run in a child process by ``test_a_loaded_job_reads_the_tag_type``: load the job -# saved under ``argv[1]`` and print each function node's tag type as JSON. +# saved under ``argv[1]`` and print each function node's stored tag type as JSON (a +# loaded node is empty until it runs, ITL-1095: its stored rows are its stored view). _READ_LOADED_TAG_TYPES = """ import json, pathlib, sys from orcapod import PipelineJob @@ -240,7 +247,7 @@ def test_a_fresh_job_reads_the_tag_type(name: str, tmp_path: pathlib.Path) -> No root = pathlib.Path(sys.argv[1]) job = PipelineJob.load(root / "job.json", database=DeltaTableDatabase(root / "store")) print(json.dumps({ - label: str(job.nodes[label].as_table().schema.field("k").type) + label: str(job.nodes[label].as_source().as_table().schema.field("k").type) for label in ("double", "increment") })) """ @@ -299,7 +306,9 @@ def build(mode: CacheMode) -> PipelineJob: build(CacheMode.LOG).run() - replayed = build(CacheMode.REPLAY).nodes["sel"].as_table() + replay_job = build(CacheMode.REPLAY) + replay_job.run() # a node shows rows only after a run (ITL-1095) + replayed = replay_job.nodes["sel"].as_table() assert replayed.schema.field("k").type == TAGS[name].type From 3301092487d2505b95a378b1295aec41e5073c6d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:40:38 -0800 Subject: [PATCH 10/48] fix(operator_node): the stored view of a loaded operator raises when its run's rows cannot be named (ITL-1095) A loaded operator node read empty before a run, so its stored view (as_source(), get_all_records) is now the way to its run's rows; it raises, as the node's reads did (ITL-1043), when the job file lacks the run's content hash and nothing is stored under the computed one. Also stamps an async REPLAY node after its inputs close, so its replayed rows are not judged stale. Loaded-operator tests read the stored view. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/operator_node.py | 77 ++++++++++------ ...26-11-13-itl-1095-node-current-run-view.md | 32 ++++++- .../loaded_job/test_loaded_operator_cache.py | 92 ++++++++++--------- .../test_pipeline/test_stored_data_context.py | 14 ++- 4 files changed, 141 insertions(+), 74 deletions(-) diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index c4d3a11f8..0c152c34f 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -1203,6 +1203,50 @@ def _make_empty_table(self) -> "pa.Table": self.data_context.type_converter, ) + def _read_run_records(self) -> "pa.Table | None": + """Read the stored rows of this node's run: its content hash, current context. + + A loaded node names its run by the content hash its job file records. A file + saved before Orcapod recorded it leaves the node only the hash the loaded job + computes; when the store holds no rows under that hash, an empty answer would + make stored results look dropped, so this raises instead (ITL-1043). + + Returns: + The run's stored rows (``_read_stored_records``), or ``None`` when the table + does not exist. + + Raises: + RuntimeError: If a loaded node's job file does not record its run's + content hash and the store holds no rows under the hash the loaded + job computes. + """ + stored = self._read_current_context_records() + records = None if stored is None else self._filter_by_content_hash(stored) + if ( + self._loaded + and self._saved_run_content_hash is None + and (records is None or records.num_rows == 0) + ): + found = ( + "is absent from the attached store" + if stored is None + else f"holds {stored.num_rows} stored rows, none of them under the " + f"content hash the loaded job computes ({self.content_hash().to_string()})" + ) + raise _LoadedCacheUnreadableError( + f"OperatorJobNode {self.label!r}: the job file does not record the " + "content hash of this node's run (a file saved before Orcapod recorded " + "it), so the loaded job cannot name the run's rows, and its cache table " + f"({'/'.join(self._stored_table_path)}) {found}. Upgrading such files is " + "left to the migration tool after v0.3 (ITL-1048); until then build the " + "job from code and attach the node with CacheMode.REPLAY to read them.", + head_note=( + "loaded node: the job file does not record its run's content hash " + "(an older file; ITL-1048)" + ), + ) + return records + def _load_cached_stream_from_db(self) -> "ArrowTableStream | None": """Read from DB in CacheMode.REPLAY only, without modifying node state. @@ -1234,31 +1278,7 @@ def _load_cached_stream_from_db(self) -> "ArrowTableStream | None": return None if self._cache_mode != CacheMode.REPLAY: return None - stored = self._read_current_context_records() - records = None if stored is None else self._filter_by_content_hash(stored) - if ( - self._loaded - and self._saved_run_content_hash is None - and (records is None or records.num_rows == 0) - ): - found = ( - "is absent from the attached store" - if stored is None - else f"holds {stored.num_rows} stored rows, none of them under the " - f"content hash the loaded job computes ({self.content_hash().to_string()})" - ) - raise _LoadedCacheUnreadableError( - f"OperatorJobNode {self.label!r}: the job file does not record the " - "content hash of this node's run (a file saved before Orcapod recorded " - "it), so the loaded job cannot name the run's rows, and its cache table " - f"({'/'.join(self._stored_table_path)}) {found}. Upgrading such files is " - "left to the migration tool after v0.3 (ITL-1048); until then build the " - "job from code and attach the node with CacheMode.REPLAY to read them.", - head_note=( - "loaded node: the job file does not record its run's content hash " - "(an older file; ITL-1048)" - ), - ) + records = self._read_run_records() if self._loaded: self._check_expected_rows(0 if records is None else records.num_rows) if records is None: @@ -1568,7 +1588,9 @@ def get_all_records( if self._pipeline_database is None: return None - results = self._read_stored_records() + # A loaded node that cannot name its run's rows raises rather than show the + # stored rows as none (ITL-1043): this is the read behind node.as_source(). + results = self._read_run_records() if self._loaded: # The advisory check of every read of a loaded node (ITL-1043). self._check_expected_rows(0 if results is None else results.num_rows) @@ -1707,6 +1729,9 @@ async def async_execute( for tag, data in self._cached_output_stream.iter_data(): await output.send((tag, data)) _ = await asyncio.gather(*discards) + # The inputs are closed, so every upstream node has stamped itself: + # stamp after them, or the replayed rows would read as stale. + self._update_modified_time() finally: for discard in discards: _ = discard.cancel() diff --git a/superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md b/superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md index de97f4096..6d8904834 100644 --- a/superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md +++ b/superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md @@ -1,6 +1,6 @@ # ITL-1095: a job node shows the current run; load-only run -Status: DIAGNOSIS + PROPOSAL sent to the coordinator (ask), waiting for decisions. No code changed. +Status: decisions received (see section 2). Implementing in four stages, one commit (or more) each, pushed to the draft PR. Base: PR #378 branch head `2e9972e0` (ITL-1093). Branch: `eywalker/itl-1095-design-what-a-job-node-shows-the-current`. Rule (developer, November 13, 2026): a job node itself (`as_table`, `iter_data`, dataframe @@ -57,9 +57,33 @@ involved in `job.run()`. Only `node.run()` reads its upstream node. Experiment (reverted): making every un-run read empty fails 400 of 4,464 job-related tests in 21 files (list in section 5). -## 2. Proposal - -See the ask message (copied in section 6 once answered). +## 2. Decisions (coordinator, November 13, 2026; proposal in 2026-11-13-itl-1095-proposal.md) + +- D1 REPLAY: R1 for the tag (after a run: union of every logged run; only the before-run read + becomes empty). ASKED of the developer. +- D2 (a): load-only uses only the node's own pipeline records; result-store-only hits are not stored. +- D3 (a)+(i): operators run on the rows that reach them (LOG writes nothing); summary and node + status mark every node below a skipped input "inputs incomplete". +- D4: orchestrator passes side-effect node rows through uncalled in load-only; node reads empty. +- D5: NodeRunSummary.not_stored, .inputs_incomplete; RunSummary.load_only; loaded rows in reused; + repr "rowwise: 6 rows: 3 loaded, 3 not stored (skipped)", "summed: 2 rows: 2 loaded (inputs incomplete: rowwise)". +- D6: amend ITL-1081 rule: run() takes no positional arguments; a job node's run() (and + job.run/run_async) takes keyword-only load_only; non-job streams' run() stays argument-less. +- D7 + developer: state-driven run. A node (itself or ancestor) that holds a completed run and is + not stale does not run again: run() returns its held view. Staleness = the existing + modified-time mechanism; a freshly built or loaded job has every node stale. Load-only-filled + is its own status (ViewStatus e.g. LOADED_ONLY, caption "N loaded, M inputs without a stored + result"), never complete: a full run re-runs it in full. A load-only run that finds everything + stays loaded-only by default (ASKED). Cascade = ancestor sub-DAG on the sync orchestrator, one + mechanism shared with job.run. Summary on node.last_run. Commits per row + flush at end. + job.run() behaviour NOT changed (give options). Side-effect re-delivery with + track_completion=False: clear_cache() then run(). Table-backed/polling sources changing without + modified time: known limit, documented. +- D8 loaded jobs as recommended. D9 as recommended. D10 remove NOTHING_STORED. D11 as recommended. + +Order: 1) empty before a run for every node kind + test updates; 2) load_only on job.run/run_async ++ summary; 3) node.run cascade + state rule + status; 4) loaded jobs. Each stage its own commit, +report counts after each. Async tests also on 3.11. Golden unchanged. ## 3. Implementation sketch (after approval) diff --git a/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py b/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py index f8a08d6a0..5a06e27b2 100644 --- a/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py +++ b/tests/test_pipeline/loaded_job/test_loaded_operator_cache.py @@ -4,10 +4,11 @@ second one and stores the join's output (``CacheMode.LOG``). Columns are declared nullable or not. A job file records each column's nullability and the run content hash of every operator node, so the loaded job reproduces each node's pipeline hash, content hash, table -path and output schema, and a loaded operator node attached in ``CacheMode.REPLAY`` reads -the rows its run stored. A file without those fields (``0.1.0``, or a ``0.2.0`` file written -before them) loads as before, lossy: the operator node still looks under its run's path, -and raises rather than show the stored rows as none. +path and output schema, and the stored view (``as_source()``) of a loaded operator node +attached in ``CacheMode.REPLAY`` reads the rows its run stored. A loaded node itself is +empty until it runs (ITL-1095). A file without those fields (``0.1.0``, or a ``0.2.0`` file +written before them) loads as before, lossy: the operator node still looks under its run's +path, and its stored view raises rather than show the stored rows as none. """ from __future__ import annotations @@ -178,17 +179,20 @@ def test_every_node_keeps_its_identity_and_schema_through_save_and_load( def test_loaded_replay_operator_reads_the_rows_its_run_stored( store_kind: str, nullability: str, read: str, tmp_path: pathlib.Path ) -> None: - """A loaded operator node in REPLAY reads exactly the rows read before saving.""" + """A loaded operator node's stored view reads exactly the rows read before saving; + the node itself is empty until it runs (ITL-1095).""" store = _store(store_kind, tmp_path) job = _run(store, nullability) expected = _sorted(job.nodes["joined"].as_table()) node = _load_replaying(_save(job, tmp_path), store).nodes["joined"] + assert node.as_table().num_rows == 0 + stored = node.as_source() if read == "as_table": - assert _sorted(node.as_table()).equals(expected) + assert _sorted(stored.as_table()).equals(expected) elif read == "iter_data": - assert len(list(node.iter_data())) == expected.num_rows + assert len(list(stored.iter_data())) == expected.num_rows else: - assert _sorted(node.head().table).equals(expected) + assert _sorted(stored.head().table).equals(expected) @pytest.mark.parametrize("nullability", list(NULLABILITY)) @@ -214,9 +218,9 @@ def rebuilt(cache_mode: CacheMode) -> PipelineJob: ) return again - assert _sorted(rebuilt(CacheMode.REPLAY).nodes["joined"].as_table()).equals( - expected - ) + replaying = rebuilt(CacheMode.REPLAY) + _ = replaying.run() # a node shows rows only after a run (ITL-1095) + assert _sorted(replaying.nodes["joined"].as_table()).equals(expected) _ = rebuilt(CacheMode.LOG).run() assert set(store.at(JOB_NAME).list_table_paths()) == tables_after_run @@ -238,7 +242,9 @@ def test_without_saved_nullability_the_operator_still_reads_its_runs_table( node = _load_replaying(saved, store).nodes["joined"] assert node.pipeline_hash() == job.nodes["joined"].pipeline_hash() assert node.node_identity_path == job.nodes["joined"].node_identity_path - assert _sorted(node.as_table()).equals(_sorted(job.nodes["joined"].as_table())) + assert _sorted(node.as_source().as_table()).equals( + _sorted(job.nodes["joined"].as_table()) + ) def _drop_both(data: dict[str, Any]) -> None: @@ -259,12 +265,13 @@ def _drop_both(data: dict[str, Any]) -> None: def test_without_the_run_content_hash_the_loaded_operator_raises( store_kind: str, nullability: str, read: str, tmp_path: pathlib.Path ) -> None: - """A node that cannot name its run's rows never shows the stored rows as none.""" + """A node that cannot name its run's rows never shows the stored rows as none: its + stored view raises.""" store = _store(store_kind, tmp_path) job = _run(store, nullability) node = _load_replaying(_save(job, tmp_path, _drop_both), store).nodes["joined"] with pytest.raises(RuntimeError, match=UNKNOWN_KEY) as excinfo: - result = getattr(node, read)() + result = getattr(node.as_source(), read)() if read == "iter_data": _ = list(result) assert "holds 2 stored rows, none of them under the content hash" in str( @@ -279,23 +286,24 @@ def test_without_the_run_content_hash_an_absent_table_raises_too( saved = _save(_run(InMemoryArrowDatabase(), "non_null"), tmp_path, _drop_both) node = _load_replaying(saved, InMemoryArrowDatabase()).nodes["joined"] with pytest.raises(RuntimeError, match=UNKNOWN_KEY) as excinfo: - _ = node.as_table() + _ = node.as_source().as_table() assert "is absent from the attached store" in str(excinfo.value) @pytest.mark.parametrize("store_kind", STORES) -def test_without_the_run_content_hash_head_says_why_it_shows_no_rows( +def test_without_the_run_content_hash_head_never_raises_and_the_stored_view_says_why( store_kind: str, tmp_path: pathlib.Path ) -> None: - """``head()`` never raises on a loaded node: it shows no rows and names the cause.""" + """``head()`` never raises on a loaded node: it has not run (ITL-1095). Its stored + view raises and names the cause.""" store = _store(store_kind, tmp_path) job = _run(store, "nullable_tag") node = _load_replaying(_save(job, tmp_path, _drop_both), store).nodes["joined"] head = node.head() - assert head.status is ViewStatus.UNAVAILABLE + assert head.status is ViewStatus.NOT_COMPUTED assert head.table.num_rows == 0 - assert "does not record its run's content hash" in head.header() - assert "ITL-1048" in head.header() + with pytest.raises(RuntimeError, match=UNKNOWN_KEY): + _ = node.as_source().head() # --------------------------------------------------------------------------- @@ -315,18 +323,19 @@ def _live_replaying(store: ArrowDatabaseProtocol, job: PipelineJob) -> Any: def _assert_reads_as_live_with_no_rows(loaded_node: Any, live_node: Any) -> None: - """The loaded node reads what a running node reads from the same store: no rows.""" + """The loaded node's stored view reads what a running node's reads from the same + store: no rows.""" + loaded_stored = loaded_node.as_source() + live_stored = live_node.as_source() for config in ({}, {"all_info": True}): - table = loaded_node.as_table(**config) - live = live_node.as_table(**config) + table = loaded_stored.as_table(**config) + live = live_stored.as_table(**config) assert table.num_rows == 0 assert table.schema.equals(live.schema), (table.schema, live.schema) - assert list(loaded_node.iter_data()) == [] - head = loaded_node.head() - assert head.status is ViewStatus.NOTHING_STORED + assert list(loaded_stored.iter_data()) == [] + head = loaded_stored.head() assert head.table.num_rows == 0 - assert "the store holds no rows for its run" in head.header() - assert live_node.head().status is ViewStatus.NOTHING_STORED + assert head.status is live_stored.head().status def test_a_store_holding_only_other_runs_rows_gives_an_empty_result( @@ -556,7 +565,8 @@ def source(column: str, values: list[int]) -> ArrowTableSource: def test_a_format_0_1_0_file_finds_its_join_table_and_never_shows_its_rows_as_none() -> ( None ): - """The real 0.1.0 file: the run's path, ``head()`` says why, ``as_table()`` raises.""" + """The real 0.1.0 file: the run's path; the node has not run (ITL-1095) and its + stored view raises.""" store = InMemoryArrowDatabase() run_join = _store_the_fixture_join(store) loaded = PipelineJob.load(str(FORMAT_0_1_0), database=store) @@ -567,13 +577,11 @@ def test_a_format_0_1_0_file_finds_its_join_table_and_never_shows_its_rows_as_no ) assert node.pipeline_hash() == run_join.pipeline_hash() assert node.node_identity_path == run_join.node_identity_path - head = node.head() - assert head.status is ViewStatus.UNAVAILABLE - assert "ITL-1048" in head.header() + assert node.head().status is ViewStatus.NOT_COMPUTED with pytest.raises( RuntimeError, match=r"'Join': the job file does not record .*ITL-1048" ): - _ = node.as_table() + _ = node.as_source().as_table() # --------------------------------------------------------------------------- @@ -596,14 +604,15 @@ def test_get_cached_output_of_a_loaded_node_reads_its_runs_rows( def test_get_cached_output_of_a_loaded_node_never_caches_unreadable_rows_as_none( tmp_path: pathlib.Path, ) -> None: - """``get_cached_output()`` raises as ``as_table()`` does; later reads raise too.""" + """``get_cached_output()`` raises as the stored view does; the node keeps nothing.""" store = InMemoryArrowDatabase() job = _run(store, "nullable_tag") node = _load_replaying(_save(job, tmp_path, _drop_both), store).nodes["joined"] with pytest.raises(RuntimeError, match="the job file does not record"): _ = node.get_cached_output() with pytest.raises(RuntimeError, match="the job file does not record"): - _ = node.as_table() + _ = node.as_source().as_table() + assert node.as_table().num_rows == 0 # --------------------------------------------------------------------------- @@ -640,11 +649,12 @@ def drop_src_a(data: dict[str, Any]) -> None: def _read_all(job: PipelineJob, label: str) -> pa.Table: - """Read a loaded node three ways and return its table.""" - node = job.nodes[label] - _ = list(node.iter_data()) - _ = node.head() - return node.as_table() + """Read a loaded node's stored view three ways and return its table (the node itself + is empty until it runs, ITL-1095).""" + stored = job.nodes[label].as_source() + _ = list(stored.iter_data()) + _ = stored.head() + return stored.as_table() def _loaded(saved: pathlib.Path, store: ArrowDatabaseProtocol) -> PipelineJob: @@ -717,7 +727,7 @@ def test_matching_counts_and_zero_expected_zero_found_do_not_warn( "joined": 0 } node = _load_replaying(empty_saved, InMemoryArrowDatabase()).nodes["joined"] - assert node.as_table().num_rows == 0 + assert node.as_source().as_table().num_rows == 0 def test_deleting_the_section_by_hand_loads_identically_without_a_check( diff --git a/tests/test_pipeline/test_stored_data_context.py b/tests/test_pipeline/test_stored_data_context.py index 49708925a..c91303ac3 100644 --- a/tests/test_pipeline/test_stored_data_context.py +++ b/tests/test_pipeline/test_stored_data_context.py @@ -764,9 +764,15 @@ def test_alone_they_give_no_rows(self, new_store: Callable[[], Store]): def test_function_node_reads_current_rows_only( self, new_store: Callable[[], Store] ): - """Every read method of a function node returns only the rows of ``CURRENT``.""" + """Every read method of a function node returns only the rows of ``CURRENT``. + + A node shows rows only after a run (ITL-1095); the run reads them from the store. + """ store = both_contexts_store(new_store) - node = build_job(store).nodes["dbl"] + job = build_job(store) + _ = job.run() + assert calls == [] + node = job.nodes["dbl"] assert rows_of(node.as_table()) == [2, 4] assert rows_of(node.get_all_records()) == [2, 4] assert sorted(d.as_dict()["double"] for _, d in node.iter_data()) == [2, 4] @@ -980,7 +986,9 @@ def test_function_node_upstream_of_a_refused_sink_is_not_refused( _ = build_job(store, with_sink=True).run() assert delivered == [] - assert rows_of(build_job(store).nodes["dbl"].as_table()) == [2, 4] + # The function node's stored rows are readable (the stored view: a node shows + # rows only after a run, ITL-1095). + assert rows_of(build_job(store).nodes["dbl"].as_source().as_table()) == [2, 4] def test_completions_logged_by_this_version_are_not_delivered_again( self, store: Store, orchestrator: Orchestrator From c8f66c029b89c702f45907edf6a40c26c4fb756f Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:42:47 -0800 Subject: [PATCH 11/48] test(loaded_job): read loaded nodes' stored results through as_source() (ITL-1095) Four tests that need the loaded node's own load path (one row per entry, EmptyData placeholders, stream columns) are strict xfails until loaded.run(load_only=True). Co-Authored-By: Claude Opus 5.5 --- docs/concepts/function-pods.md | 4 +- .../loaded_job/test_loaded_job_readback.py | 102 ++++++++++++------ .../loaded_job/test_loaded_logical_schemas.py | 3 +- 3 files changed, 72 insertions(+), 37 deletions(-) diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 5f414b039..5a0d89d7c 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -687,7 +687,9 @@ job.run() job.save(workdir / "bmi_job.json") loaded = PipelineJob.load(workdir / "bmi_job.json") -print(loaded.nodes["bmi"].as_table().sort_by("subject_id").to_pandas()) +print(loaded.nodes["bmi"].as_table().num_rows) # not run in this process: empty +# 0 +print(loaded.nodes["bmi"].as_source().as_table().sort_by("subject_id").to_pandas()) # subject_id bmi # 0 mouse_01 1756.944444 # 1 mouse_02 2210.000000 diff --git a/tests/test_pipeline/loaded_job/test_loaded_job_readback.py b/tests/test_pipeline/loaded_job/test_loaded_job_readback.py index 962131eb7..50a54b71c 100644 --- a/tests/test_pipeline/loaded_job/test_loaded_job_readback.py +++ b/tests/test_pipeline/loaded_job/test_loaded_job_readback.py @@ -142,7 +142,8 @@ def record() -> None: path = _save_small_job(tmp_path, record) node = PipelineJob.load(path).nodes["stamp"] assert dict(node.output_schema()[1]) == {"t": datetime.datetime} - assert node.as_table().num_rows == 2 + # A loaded node is empty until it runs (ITL-1095); its stored view reads the rows. + assert node.as_source().as_table().num_rows == 2 def test_datetime_output_with_nothing_stored_returns_typed_zero_rows( @@ -189,8 +190,9 @@ def record() -> None: node = PipelineJob.load(_save_small_job(tmp_path, record)).nodes["paths"] assert node.output_schema()[1]["p"] is pathlib.Path - # The stored rows are still readable, with their stored logical type. - table = node.as_table() + # The stored rows are still readable, with their stored logical type (through the + # stored view: a loaded node is empty until it runs, ITL-1095). + table = node.as_source().as_table() assert table.num_rows == 2 assert isinstance(table.schema.field("p").type, pa.ExtensionType) @@ -219,20 +221,25 @@ def record() -> None: Join()(paths, other, label="joined") loaded = PipelineJob.load(_save_small_job(tmp_path, record)) - assert loaded.nodes["paths"].as_table().num_rows == 2 + assert loaded.nodes["paths"].as_source().as_table().num_rows == 2 @pytest.mark.parametrize("label", FUNCTION_NODES) def test_loaded_function_node_as_table_returns_stored_rows( saved: pathlib.Path, label: str ) -> None: - """A loaded node's table equals the live table: schema and values.""" + """A loaded node's stored view equals the live table: schema and values.""" expected = pq.read_table(saved / "expected" / f"{label}.parquet") - table = _load(saved).nodes[label].as_table() + table = _load(saved).nodes[label].as_source().as_table() assert table.schema == expected.schema assert _sorted(table, TAG_OF[label]).equals(_sorted(expected, TAG_OF[label])) +@pytest.mark.xfail( + strict=True, + reason="ITL-1095: a loaded node reads empty until loaded.run(load_only=True), " + "which reads its entries this way (stage 4)", +) @pytest.mark.parametrize("label", FUNCTION_NODES) def test_loaded_function_node_as_table_all_info_has_stream_columns( saved: pathlib.Path, label: str @@ -243,7 +250,7 @@ def test_loaded_function_node_as_table_all_info_has_stream_columns( # source-info columns. (The live FunctionJobNode.as_table() drops those two # groups -- a separate, pre-existing bug the loaded node does not copy.) expected = pq.read_table(saved / "expected" / f"{label}.all_info.parquet") - table = _load(saved).nodes[label].as_table(all_info=True) + table = _load(saved).nodes[label].as_source().as_table(all_info=True) data_column = next(iter(_load(saved).nodes[label].output_schema()[1])) assert "_context_key" in table.column_names assert f"_source_{data_column}" in table.column_names @@ -258,7 +265,7 @@ def test_loaded_function_node_iter_data_yields_stored_rows(saved: pathlib.Path) rows = sorted( ( (tag.as_dict(), data.as_dict()) - for tag, data in _load(saved).nodes["dbl"].iter_data() + for tag, data in _load(saved).nodes["dbl"].as_source().iter_data() ), key=lambda row: row[0]["id"], ) @@ -373,24 +380,32 @@ def test_other_context_records_do_not_satisfy_saved_expectation( node = PipelineJob.load(path, database=other).nodes["dbl"] # The store is the source of truth (ITL-1043): it holds none of the run's # records, so none is read; the job file's count makes it a warning. + # A loaded node is empty until it runs (ITL-1095): its stored view reads the store. + reader = node if read_method == "get_all_records" else node.as_source() with pytest.warns(StoreContentWarning, match="records 2 rows .* holds 0"): - result = getattr(node, read_method)() + result = getattr(reader, read_method)() rows = list(result) if read_method == "iter_data" else result assert rows is None or len(rows) == 0 +@pytest.mark.xfail( + strict=True, + reason="ITL-1095: a loaded node reads empty until loaded.run(load_only=True), " + "which reads its entries this way (stage 4)", +) def test_recomputed_entry_is_read_once_with_its_latest_result( tmp_path: pathlib.Path, ) -> None: """A loaded node reads one row per entry: the greatest recomputation index wins.""" path = _save_small_job(tmp_path, _record_double) node = PipelineJob.load(path).nodes["dbl"] - entry_id = next(iter(node.iter_data()))[0].as_dict()["id"] + # A loaded node is empty until it runs (ITL-1095): read its stored view. + entry_id = next(iter(node.as_source().iter_data()))[0].as_dict()["id"] # Index 2 is stored before index 1, so "last record wins" would pick 111. _append_recomputed_record(node, entry_id, recomputation_index=2, value=222) _append_recomputed_record(node, entry_id, recomputation_index=1, value=111) - reloaded = PipelineJob.load(path).nodes["dbl"] + reloaded = PipelineJob.load(path).nodes["dbl"].as_source() table = reloaded.as_table() assert table.num_rows == 2 rows = {row["id"]: row["dbl"] for row in table.to_pylist()} @@ -418,6 +433,8 @@ def test_live_node_hot_load_reads_the_latest_recomputed_result( fresh = PipelineJob(name="small", database=store) with fresh: _record_double() + # A node shows rows only after a run (ITL-1095): the run loads every stored entry. + _ = fresh.run() values = { t.as_dict()["id"]: d.as_dict()["dbl"] for t, d in fresh.nodes["dbl"].iter_data() } @@ -425,6 +442,11 @@ def test_live_node_hot_load_reads_the_latest_recomputed_result( assert values[entry_id] == 222 +@pytest.mark.xfail( + strict=True, + reason="ITL-1095: a loaded node reads empty until loaded.run(load_only=True), " + "which reads its entries this way (stage 4)", +) def test_loaded_node_with_ephemeral_results_does_not_crash( tmp_path: pathlib.Path, ) -> None: @@ -460,6 +482,11 @@ def make_ephemeral(job: PipelineJob) -> None: ) +@pytest.mark.xfail( + strict=True, + reason="ITL-1095: a loaded node reads empty until loaded.run(load_only=True), " + "which reads its entries this way (stage 4)", +) def test_newer_unavailable_result_does_not_expose_old_persistent_result( tmp_path: pathlib.Path, ) -> None: @@ -503,11 +530,14 @@ def test_loaded_function_node_warns_once_on_a_store_without_its_runs_records( issued once per node. """ node = _load(saved, database=InMemoryArrowDatabase()).nodes["dbl"] + # A loaded node is empty until it runs (ITL-1095): the store is read through its + # stored view, which warns once. + stored = node.as_source() with pytest.warns(StoreContentWarning) as record: - table = node.as_table() - rows = list(node.iter_data()) + table = stored.as_table() + rows = list(stored.iter_data()) records = node.get_all_records() - head = node.head() + head = stored.head() assert table.num_rows == 0 assert table.column_names == ["id", "dbl"] assert rows == [] @@ -532,7 +562,7 @@ def test_store_content_warning_names_a_delta_stores_location( other = tmp_path / "other_store" node = _load(saved, database=DeltaTableDatabase(str(other))).nodes["dbl"] with pytest.warns(StoreContentWarning, match="the store at '") as record: - _ = node.as_table() + _ = node.as_source().as_table() assert str(other) in str(record[0].message) @@ -568,7 +598,7 @@ def double(x: int) -> int: assert node._pipeline_database is not None assert node._pipeline_database.table_exists(node._versioned_pipeline_path) with pytest.warns(StoreContentWarning, match="records 2 rows .* holds 0"): - assert node.as_table().num_rows == 0 + assert node.as_source().as_table().num_rows == 0 def test_saved_file_records_the_rows_each_node_had_in_the_store( @@ -598,7 +628,7 @@ def test_saving_a_loaded_job_keeps_the_expected_rows( assert run["expected_rows"] == original node = PipelineJob.load(str(resaved), database=InMemoryArrowDatabase()).nodes["dbl"] with pytest.warns(StoreContentWarning, match="records 2 rows"): - assert node.as_table().num_rows == 0 + assert node.as_source().as_table().num_rows == 0 def test_loaded_function_node_with_nothing_stored_returns_zero_rows( @@ -634,8 +664,9 @@ def drop_section(data: dict[str, Any]) -> None: with warnings.catch_warnings(): warnings.simplefilter("error", StoreContentWarning) job = PipelineJob.load(path, database=InMemoryArrowDatabase()) - assert job.nodes["dbl"].as_table().num_rows == 0 - assert PipelineJob.load(path).nodes["dbl"].as_table().num_rows == 2 + assert job.nodes["dbl"].as_source().as_table().num_rows == 0 + stored = PipelineJob.load(path).nodes["dbl"].as_source() + assert stored.as_table().num_rows == 2 def test_a_file_with_the_old_stored_records_flag_loads_without_a_check( @@ -689,7 +720,7 @@ def drop_store(data: dict[str, Any]) -> None: job = PipelineJob.load(_rewrite_job_file(saved, tmp_path, drop_store)) job.bind(database=DeltaTableDatabase(str(saved / "store"))) - assert job.nodes["dbl"].as_table().num_rows == 2 + assert job.nodes["dbl"].as_source().as_table().num_rows == 2 def test_loaded_job_still_cannot_run(saved: pathlib.Path) -> None: @@ -799,21 +830,19 @@ async def run() -> list[Any]: # --------------------------------------------------------------------------- -def test_loaded_operator_node_raises_instead_of_returning_empty( +def test_loaded_operator_node_reads_empty_and_says_it_has_not_run( saved: pathlib.Path, ) -> None: - """A loaded operator node, with nothing stored, raises and names its inputs.""" + """A loaded operator node, like every node, is empty until it runs (ITL-1095): no + read raises, and its head points to ``run()`` and ``as_source()``.""" + from orcapod.core.streams import ViewStatus + node = _load(saved).nodes["joined"] - expected = ( - "OperatorJobNode 'joined' has no stored results available for this read. " - "Operator outputs require a stored cache and replay configuration. Read the stored results of " - "its input nodes ('dbl', 'asrc') instead; an input source whose data was not " - "saved must be bound first with job.bind(sources=...)." - ) - for read in (node.as_table, lambda: list(node.iter_data())): - with pytest.raises(RuntimeError) as excinfo: - _ = read() - assert str(excinfo.value) == expected + assert node.as_table().num_rows == 0 + assert list(node.iter_data()) == [] + head = node.head() + assert head.status is ViewStatus.NOT_COMPUTED + assert "as_source()" in head.header() # --------------------------------------------------------------------------- @@ -867,7 +896,7 @@ def move_csv(data: dict[str, Any]) -> None: "source node 'csrc' itself, bind a source: job.bind(sources={'csrc': })." ) # The stored results downstream of the missing source are still readable. - assert job.nodes["inc"].as_table().num_rows == 2 + assert job.nodes["inc"].as_source().as_table().num_rows == 2 def test_unreconstructable_store_is_a_warning_with_the_remedy( @@ -976,5 +1005,8 @@ def timestamp(x: int) -> int: assert result is not None expected = result["value"] node.clear_cache() - assert node.as_table().column("value").to_pylist() == [expected] - assert [result["value"] for _, result in node.iter_data()] == [expected] + # A cleared node is empty until it runs (ITL-1095); the readback of the entry, the + # lookup a run makes, picks the newest recomputation from whichever store holds it. + assert node.as_table().num_rows == 0 + loaded = node.get_cached_results([node.compute_base_entry_id(tag, data)]) + assert [result["value"] for _, result in loaded.values()] == [expected] diff --git a/tests/test_pipeline/loaded_job/test_loaded_logical_schemas.py b/tests/test_pipeline/loaded_job/test_loaded_logical_schemas.py index b4b90b38d..09f204881 100644 --- a/tests/test_pipeline/loaded_job/test_loaded_logical_schemas.py +++ b/tests/test_pipeline/loaded_job/test_loaded_logical_schemas.py @@ -127,7 +127,8 @@ def test_registered_logical_output_keeps_join_available( loaded = op.PipelineJob.load(path) assert loaded.nodes["value"].output_schema()[1]["value"] is expected_type - assert loaded.nodes["value"].as_table().num_rows == 1 + # A loaded node is empty until it runs (ITL-1095): read its stored view. + assert loaded.nodes["value"].as_source().as_table().num_rows == 1 assert loaded.nodes["joined"].load_status == LoadStatus.FULL assert loaded.nodes["joined"].output_schema()[1]["value"] is expected_type From da66b836fd8fa025345131614c5e3841c3f65795 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:45:13 -0800 Subject: [PATCH 12/48] docs: a job node shows its current run, as_source() what is stored (ITL-1095) Concept pages, PipelineJob.load docstring, CLAUDE.md and .zed/rules rule (replaces the ITL-1093 sentence), changelog fragment and design issue CR1. Staleness across the job graph moves into a RunHoldingNode mixin (no import cycle). Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 18 +++-- CLAUDE.md | 18 +++-- changelog.d/ITL-1095.md | 9 +++ design_issues/ITL-1095-CR1.md | 24 +++++++ docs/concepts/function-pods.md | 69 ++++++++++--------- docs/concepts/sources.md | 4 +- docs/concepts/streams.md | 9 +-- src/orcapod/core/nodes/function_node.py | 23 ++----- src/orcapod/core/nodes/operator_node.py | 15 +--- src/orcapod/core/nodes/staleness.py | 33 ++++++--- src/orcapod/pipeline/job.py | 13 ++-- .../loaded_job/test_loaded_job_readback.py | 3 +- 12 files changed, 144 insertions(+), 94 deletions(-) create mode 100644 changelog.d/ITL-1095.md create mode 100644 design_issues/ITL-1095-CR1.md diff --git a/.zed/rules b/.zed/rules index 2d923a544..6653ea945 100644 --- a/.zed/rules +++ b/.zed/rules @@ -773,13 +773,19 @@ data. The rules: by the user is never changed. SourceStreamBuilder.build has no bare-hash default: it needs a source_id or a source_kind. - ArrowTableStream requires at least one data column; raises ValueError otherwise. -- After a run, a FunctionJobNode returns exactly the rows of that run's inputs, sync or - async (ITL-1093; a run that raised defines no rows). Read without a completed run in this - process (a fresh or loaded job), it returns every record stored under its content hash for - the current data context, not only its current inputs. A run skips inputs whose entry is - already stored. +- A job node shows its current run; node.as_source() shows everything stored for it + (developer, November 13, 2026, ITL-1095). After a run, a FunctionJobNode returns exactly + the rows of that run's inputs, sync or async (ITL-1093). Before a run in this process (a + fresh or loaded job), after a run that raised, after clear_cache(), and while stale, every + function and operator job node reads empty (zero typed rows; head() is not_computed and + points to run() and as_source()), in every cache mode: no read touches the store. A + never-run or cleared node has no modified time, so it is stale; a job node is stale also + when any function or operator job node it depends on is (core/nodes/staleness.py), and a + SourceJobNode is modified when it is bound or its bound source is. A run skips inputs + whose entry is already stored. A loaded operator's stored view raises when its run's rows + cannot be named (ITL-1043). - Empty data → ArrowTableSource raises ValueError("Table is empty"). -- DerivedSource over a job node before run() → presents zero rows with the origin's columns +- DerivedSource over a job node with nothing stored → presents zero rows with the origin's columns (no computed records; no error), in the plain form and with tags_as_data. - Join requires non-overlapping data columns; raises InputValidationError on collision. - MergeJoin requires colliding data columns to have identical Python types; merges into sorted diff --git a/CLAUDE.md b/CLAUDE.md index d075910b4..66a284382 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -814,13 +814,19 @@ data. The rules: by the user is never changed. `SourceStreamBuilder.build` has no bare-hash default: it needs a `source_id` or a `source_kind`. - `ArrowTableStream` requires at least one data column; raises `ValueError` otherwise. -- After a run, a `FunctionJobNode` returns exactly the rows of that run's inputs, sync or - async (ITL-1093; a run that raised defines no rows). Read without a completed run in this - process (a fresh or loaded job), it returns every record stored under its content hash for - the current data context, not only its current inputs. A run skips inputs whose entry is - already stored. +- A job node shows its current run; `node.as_source()` shows everything stored for it + (developer, November 13, 2026, ITL-1095). After a run, a `FunctionJobNode` returns exactly + the rows of that run's inputs, sync or async (ITL-1093). Before a run in this process (a + fresh or loaded job), after a run that raised, after `clear_cache()`, and while stale, every + function and operator job node reads empty (zero typed rows; `head()` is `not_computed` and + points to `run()` and `as_source()`), in every cache mode: no read touches the store. A + never-run or cleared node has no modified time, so it is stale; a job node is stale also + when any function or operator job node it depends on is (`core/nodes/staleness.py`), and a + `SourceJobNode` is modified when it is bound or its bound source is. A run skips inputs + whose entry is already stored. A loaded operator's stored view raises when its run's rows + cannot be named (ITL-1043). - Empty data → `ArrowTableSource` raises `ValueError("Table is empty")`. -- `DerivedSource` over a job node before `run()` → presents zero rows with the origin's columns +- `DerivedSource` over a job node with nothing stored → presents zero rows with the origin's columns (no computed records; no error), in the plain form and with `tags_as_data`. - Join requires non-overlapping data columns; raises `InputValidationError` on collision. - MergeJoin requires colliding data columns to have identical Python types; merges into sorted diff --git a/changelog.d/ITL-1095.md b/changelog.d/ITL-1095.md new file mode 100644 index 000000000..0a5c89eeb --- /dev/null +++ b/changelog.d/ITL-1095.md @@ -0,0 +1,9 @@ +### Breaking changes and migration + +- A job node shows the current run; `node.as_source()` shows everything stored for it (developer, November 13, 2026; ITL-1095). Before a run in this process, a function or operator job node is empty: `as_table()`, `iter_data()`, the dataframe methods, `flow()`, `head()` and the notebook preview return zero rows with the node's columns, and `head()` (status `not_computed`) says the node has not run and points to `run()` and to `node.as_source()`. Before, a function node built over a store that already held results, or loaded with `PipelineJob.load`, returned every row stored under its key, and an operator node in `CacheMode.REPLAY` returned every logged run. The same holds after `clear_cache()`, after a run that raised, and while the node is stale. Migration: read stored results with `node.as_source()` (or `get_all_records()`), or run the job first; a run over a store that holds every result computes nothing. +- A function or operator job node that has not run in this process is stale (`is_stale` is `True`; it has no modified time), and a job node is stale also when any function or operator job node it depends on is. Binding a source to a job's source slot (`job.bind(sources=...)`) marks the slot modified, so every node fed by it is stale and empty until it runs again. +- A loaded operator node no longer raises from `as_table()` or `iter_data()`: it is empty, like every node that has not run. Its stored view (`as_source()`, `get_all_records()`) raises instead when the job file does not record its run's content hash and the store holds nothing under the hash the loaded job computes (ITL-1043). + +### Fixed + +- On the asynchronous orchestrator, an operator node in `CacheMode.REPLAY` stamps itself after its inputs close, so the rows it replayed are not judged out of date by the time the run ends. diff --git a/design_issues/ITL-1095-CR1.md b/design_issues/ITL-1095-CR1.md new file mode 100644 index 000000000..b2e853b2c --- /dev/null +++ b/design_issues/ITL-1095-CR1.md @@ -0,0 +1,24 @@ +## `src/orcapod/core/nodes/function_node.py` + +### CR1 — A job node read without a run showed every stored row, not the current run +**Status:** in progress +**Severity:** high +**Issue:** ITL-1095 + +A job node is read through the same methods whether or not it has run in this process. After +a run it held the rows of that run's inputs (ITL-1093), but read without one it fell back to the +store: a `FunctionJobNode` hot-loaded every record stored under its content hash (the results of +every earlier run over the same `source_id`, including inputs no longer in the source), a loaded +job's function nodes did the same, and an `OperatorJobNode` in `CacheMode.REPLAY` returned every +logged run. Which view a read gave depended on whether a run had happened in the process, and +nothing said so. A fresh or cleared node also reported `is_stale == False`, and staleness looked +one level up only, so a source rebound two nodes up left the nodes below it "current". + +**Fix:** (developer, November 13, 2026) a job node shows its current run and `node.as_source()` +shows everything stored. Function and operator job nodes read empty (typed zero rows, `head()` +`not_computed` pointing to `run()` and `as_source()`) until a run in this process, after +`clear_cache()`, after a run that raised and while stale; no read touches the store. A never-run +or cleared node has no modified time (stale); `core/nodes/staleness.py` makes a job node stale +when any function or operator job node above it is; a `SourceJobNode` is modified when bound or +when its bound source is. Planned in the same issue: `run(load_only=True)` and the `node.run()` +cascade. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 5a0d89d7c..d32956901 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -333,12 +333,19 @@ database. Each recorded step becomes a node, available afterwards as `job.nodes[ listed under its `source_id`). A node is a stream, so `as_table()`, `iter_data()` and `output_schema()` all work on it. -After `job.run()`, a function node returns exactly the rows of that run's inputs, on either -orchestrator. The database may hold more under the same node: results of an earlier run -over other rows of the source stay stored and are reused when those rows come back, but -they are not rows of this run. A run that raised defines no rows. A node read with no -finished run in this process (a job built but not run, or a loaded job) returns every row -stored under its key. +A job node shows the current run; `node.as_source()` shows everything stored for it +(developer, November 13, 2026). After `job.run()`, a function node returns exactly the rows +of that run's inputs, on either orchestrator. The database may hold more under the same +node: results of an earlier run over other rows of the source stay stored and are reused +when those rows come back, but they are not rows of this run. Before a run in this process +(a job built over a store that already holds results, or a loaded job) every node is empty: +`as_table()`, `iter_data()`, `head()` and the preview show no rows, and the preview says the +node has not run and points to `run()` and to `as_source()`. A run fills it, also when every +result is read from the store and nothing is computed. A run that raised defines no rows, +and so does `clear_cache()`; a node whose input changed since its run (an upstream node ran +again, or a source was bound again) is stale and empty until it runs again. The stored view, +`node.as_source()`, reads every record stored under the node's key, whether or not the node +has run. `InMemoryArrowDatabase` lasts only as long as the Python process. Use `DeltaTableDatabase("some/directory")` from `orcapod.databases` to keep results on disk @@ -686,7 +693,7 @@ with job: job.run() job.save(workdir / "bmi_job.json") -loaded = PipelineJob.load(workdir / "bmi_job.json") +loaded = PipelineJob.load(workdir / "bmi_job.json") # its nodes are empty until it runs print(loaded.nodes["bmi"].as_table().num_rows) # not run in this process: empty # 0 print(loaded.nodes["bmi"].as_source().as_table().sort_by("subject_id").to_pandas()) @@ -696,29 +703,29 @@ print(loaded.nodes["bmi"].as_source().as_table().sort_by("subject_id").to_pandas # 2 mouse_03 2297.520661 ``` -A loaded job is for reading stored results. It cannot be run again: function pods are not -saved with the job, so `loaded.run()` raises. To compute again, build the job from your code -as before. - -The store is the source of truth: a loaded job shows what the store holds for it. The job -file names each run's rows by the content hash the run stored them under, so a loaded node -reads exactly what the same node built from your code would read from that store, and no -rows under that key is an empty result, not an error. - -- A function node's `as_table()`, `iter_data()` and `get_all_records()` return the rows - stored under the saved run's key, with the types they were stored with: those of every - run of that node over the same sources, not only the saved run's inputs. Rows come in the order the store - returns them; sort them if order matters. +A loaded job is for reading stored results, through each node's stored view, +`node.as_source()`: its nodes have not run in this process, so they are empty. It cannot be +run again: function pods are not saved with the job, so `loaded.run()` raises. To compute +again, build the job from your code as before. + +The store is the source of truth: a loaded job's stored views show what the store holds for +it. The job file names each run's rows by the content hash the run stored them under, so a +loaded node's stored view reads exactly what the same node built from your code would read +from that store, and no rows under that key is an empty result, not an error. + +- A function node's `as_source()` and `get_all_records()` return the rows stored under the + saved run's key, with the types they were stored with: those of every run of that node + over the same sources, not only the saved run's inputs. Rows come in the order the store + returns them; sort them if order matters. The node itself (`as_table()`, `iter_data()`, + `head()`) is empty: it has not run in this process. - Operators (a join, a filter) store outputs only when caching is enabled, under `odb_v1`. The saved file does not record an operator's cache mode (follow-up ITL-977), so attach a loaded operator node in `CacheMode.REPLAY` to read the rows its run stored: `loaded.nodes["joined"].attach_databases(pipeline_database=store.at(""), - cache_mode=CacheMode.REPLAY)`. A loaded operator node reads exactly as the same node built - from your code and attached in `CacheMode.REPLAY`; when the store holds no rows under the - node's key (no table, or none of its rows), the answer is an empty result, as for that - node. A loaded operator not attached in `REPLAY` has no readable cache and raises from - `as_table()` and `iter_data()`; read its input nodes instead (an in-memory input source - must be bound first, see below), or build the job from your code. + cache_mode=CacheMode.REPLAY)`, then read its stored view, `as_source()`, which reads + exactly as the same node built from your code would; when the store holds no rows under + the node's key (no table, or none of its rows), the answer is an empty result, as for that + node. A loaded operator node itself is empty, like every node that has not run. - A loaded node has the pipeline hash, content hash, table path and output schema its run had (a source node whose data was not saved gets its run's content hash once its source is bound). The file records which columns may be null (`T | None`), because the Arrow type text @@ -727,8 +734,8 @@ rows under that key is an empty result, not an error. file's format version: a file without them (format `0.1.0`, or a `0.2.0` file written before they existed) still loads, but lossy: its schemas read every column as non-null, and a loaded operator node cannot name its run's rows. When the store holds none under the - content hash the loaded job computes, `as_table()` and `iter_data()` raise, and `head()` - shows no rows and says why, rather than presenting stored rows as none. An empty record is + content hash the loaded job computes, its stored view (`as_source()`, `get_all_records()`) + raises and says why, rather than presenting stored rows as none. An empty record is not a missing one: it says that no column may be null. Saving such a loaded job again does not record what its file did not. Upgrading such files is left to a migration tool after v0.3. @@ -741,9 +748,9 @@ rows under that key is an empty result, not an error. `PipelineJob.load(path, database=...)` if the store has moved. - The job file may also hold an optional, advisory section, `run.expected_rows`: for each node, how many rows the store held under its run's key when the job was saved (written - only when that is certain: after a run that finished). Reading a loaded node always - returns what the store holds; when the store holds another number of rows, the read - issues one `StoreContentWarning` per node, naming the node, both numbers and the store: + only when that is certain: after a run that finished). Reading a loaded node's stored + view always returns what the store holds; when the store holds another number of rows, + the read issues one `StoreContentWarning` per node, naming the node, both numbers and the store: the store may not be the one the run wrote, or it may have changed since. Deleting the section from the file changes nothing but this check. diff --git a/docs/concepts/sources.md b/docs/concepts/sources.md index 83d4a8319..b24d1fc6f 100644 --- a/docs/concepts/sources.md +++ b/docs/concepts/sources.md @@ -167,8 +167,8 @@ names them. To see more rows, use `as_df()` with a `limit=` or `where=` source. **Restrictions are not part of identity.** `where`, `limit` and `asof` choose rows of the same table, so two sources on one table have the same `source_id` and `content_hash` whatever their restrictions, and a row's result is computed once and shared. One -consequence: with a shared store, reading a job's node back (`get_all_records()`, a loaded -job, and `as_table()` after an async run) returns every row that any job with the same +consequence: with a shared store, reading a job node's stored records back +(`get_all_records()`, `node.as_source()`) returns every row that any job with the same pipeline has read from that table, not only the rows of the latest restriction. The rows sent downstream during a run are only the rows that run read. diff --git a/docs/concepts/streams.md b/docs/concepts/streams.md index 7d0250146..838cc31fe 100644 --- a/docs/concepts/streams.md +++ b/docs/concepts/streams.md @@ -205,7 +205,7 @@ What `run()` does on each kind of object: | A `PollingSource` | polls, as `as_table()` does | | A `CachedSource` | reads its cache and writes the wrapped source's new rows to it | | A function pod's or operator's output stream | computes what is not computed yet | -| A job node | runs the node as a job run would (`as_table()` of a job node only reads), then returns the view | +| A job node | runs the node as a job run would (`as_table()` of a job node only reads), then returns the view of that run's rows | | A side-effect stream | delivers its side effects for every row, on every call, and returns the rows it passed on; `head()` still shows none | | A pipeline blueprint node, an unbound input slot | raises, as `as_table()` does | @@ -215,7 +215,7 @@ What each kind of object shows: |---|---| | A source | its first rows (a bounded read for a source that reads its table lazily) | | An operator's or function pod's output stream | the rows already computed, or none with status `not_computed` | -| A job node | the rows it holds after `job.run()`, else the first of its stored rows, else none (`nothing_stored`) | +| A job node | the rows of its current run (after `job.run()` or `node.run()`); before a run in this process, or once stale, none (`not_computed`), pointing to `run()` and to `node.as_source()`, the view of every stored result | | A pipeline blueprint node, an unbound input slot | none (`unavailable`) | | A side-effect stream | none: showing it never delivers its side effects | @@ -240,7 +240,7 @@ print(unordered.head(2, all_info=True).as_table().column("subject_id").to_pylist # ['mouse_01', 'mouse_02'] ``` -A stream that has all of its rows (a job node's stored rows, a computed stream, an in-memory +A stream that has all of its rows (a job node after its run, a computed stream, an in-memory source) then gives exactly the first `n` rows of `as_table` with the same configuration, whatever order the rows were computed in. A stream that has only some of its rows sorts those: a source that reads a bounded number of rows (such as a `SpiralDBTableSource`) shows @@ -260,7 +260,8 @@ rows, can be missing from it even when `columns` or `all_info` asks for them. A stream or node records that its results are current when it has read its input to the end. So after `job.run()` on either orchestrator, sync or async, every node shows the rows of that run: `head()`, `as_table()` and `iter_data()` serve them from memory, without reading the -store (a function node whose run left no rows in memory reads its stored rows). Likewise, once a chain of streams such as `g.pod(f.pod(source))` has been read to the +store. Before its run in this process a job node holds nothing and shows no rows, whatever +its store holds; `node.as_source()` reads the store. Likewise, once a chain of streams such as `g.pod(f.pod(source))` has been read to the end, reading it again computes nothing. A change upstream (`clear_cache()`) makes every stream after it out of date, and its next read computes it again. Two reads of one stream open at the same time, or a read that picks up one left partway, share diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 7b28f3b4d..cb8a5ced9 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -42,7 +42,7 @@ StreamView, ) from orcapod.core.tracker import DEFAULT_TRACKER_MANAGER -from orcapod.core.nodes.staleness import upstream_job_node_stale +from orcapod.core.nodes.staleness import RunHoldingNode from orcapod.core.datagrams.tag_data import Data, EmptyData, Tag from orcapod.errors import ( EphemeralResultMissingError, @@ -1035,7 +1035,7 @@ class _JoinedRecords(NamedTuple): empty_taginfo_rows: dict[bytes, dict] -class FunctionJobNode(FunctionNodeBase): +class FunctionJobNode(RunHoldingNode, FunctionNodeBase): """DB-backed execution node for function pod invocations. Created by ``PipelineJob`` at run time; never recorded inside a plain @@ -1529,17 +1529,6 @@ def clear_cache(self) -> None: # No content established: the node, and every node downstream, is stale. self._set_modified_time(None) - @property - def is_stale(self) -> bool: - """Whether the rows this node holds are out of date, or it holds none. - - True before the node's first run in this process and after ``clear_cache()`` - (no modified time), when an input or the function pod was modified after the node's - run (``StreamBase.is_stale``), or when any job node it depends on is stale - (ITL-1095). A stale node reads empty until it runs again. - """ - return super().is_stale or upstream_job_node_stale(self.upstreams) - def _held_pairs(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: """Yield the (tag, data) rows this node holds; every read of them goes through here. @@ -1626,7 +1615,7 @@ def _cannot_compute_error(self) -> RuntimeError: return RuntimeError( f"FunctionJobNode {self.label!r} was loaded by PipelineJob.load and has no " "function pod, so it cannot compute; read its stored results with " - "as_table() or iter_data()." + "as_source()." ) def _stored_record_count(self) -> int: @@ -3241,8 +3230,8 @@ def run(self) -> StreamView: Computes as a job run computes this node, unlike ``as_table()``, which only reads. A node whose upstream is unavailable (``CACHE_ONLY``) or whose function - pod is a placeholder (``READ_ONLY``) computes nothing: the view holds its stored - rows. + pod is a placeholder (``READ_ONLY``) computes nothing and holds no rows; its + stored rows are read with ``as_source()``. Returns: A ``StreamView`` of every row ``as_table()`` then returns. It says "rows", @@ -3263,7 +3252,7 @@ def run(self) -> StreamView: if self._load_status in (LoadStatus.CACHE_ONLY, LoadStatus.READ_ONLY): # CACHE_ONLY: upstream unavailable; computation requires a live input stream. # READ_ONLY: function pod is a proxy placeholder — cannot compute. - # The view serves the existing DB results, as iter_data() does. + # Neither runs, so the view holds no rows (ITL-1095). return self._view_of_rows(self.as_table(), "held") if self.is_stale: # Discard any stale in-memory entries before a fresh computation run diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 0c152c34f..7ca6367d4 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -40,7 +40,7 @@ head_of, ) from orcapod.core.tracker import DEFAULT_TRACKER_MANAGER -from orcapod.core.nodes.staleness import upstream_job_node_stale +from orcapod.core.nodes.staleness import RunHoldingNode from orcapod.errors import ( PipelineJobRequiredError, StoreReadError, @@ -621,7 +621,7 @@ def as_node(self) -> OperatorNode: # --------------------------------------------------------------------------- -class OperatorJobNode(OperatorNodeBase): +class OperatorJobNode(RunHoldingNode, OperatorNodeBase): """DB-backed execution node for operator pod invocations. Stream node representing an operator invocation with optional DB persistence. @@ -1480,17 +1480,6 @@ def run(self) -> StreamView: origin = "computed" if kept is None else head_of(kept.head(1), self).origin return self._view_of_rows(table, origin) - @property - def is_stale(self) -> bool: - """Whether the rows this node holds are out of date, or it holds none. - - True before the node's first run in this process and after ``clear_cache()`` - (no modified time), when an input or the operator was modified after the node's - run (``StreamBase.is_stale``), or when any job node it depends on is stale - (ITL-1095). A stale node reads empty until it runs again. - """ - return super().is_stale or upstream_job_node_stale(self.upstreams) - def _current_output(self) -> StreamProtocol | None: """The output of the node's current run, or ``None`` when it holds none. diff --git a/src/orcapod/core/nodes/staleness.py b/src/orcapod/core/nodes/staleness.py index b5320cce4..06efbe5b8 100644 --- a/src/orcapod/core/nodes/staleness.py +++ b/src/orcapod/core/nodes/staleness.py @@ -12,23 +12,40 @@ from typing import Any +class RunHoldingNode: + """Mixin of the job nodes that hold the rows of their current run. + + ``FunctionJobNode`` and ``OperatorJobNode`` mix it in ahead of their stream base, so + their ``is_stale`` covers their whole ancestry. Side-effect job nodes do not yet: + their own ``is_stale`` is redefined by ITL-1091. + """ + + @property + def is_stale(self) -> bool: + """Whether the rows this node holds are out of date, or it holds none. + + True before the node's first run in this process and after ``clear_cache()`` + (no modified time), when an input or its producer was modified after the + node's run (``StreamBase.is_stale``), or when any job node it depends on is + stale (ITL-1095). A stale node reads empty until it runs again. + """ + own: bool = super().is_stale # pyright: ignore[reportAttributeAccessIssue] # the stream base after this mixin in the MRO defines it + return own or upstream_job_node_stale(self.upstreams) # pyright: ignore[reportAttributeAccessIssue] # every node class this is mixed into has upstreams + + def upstream_job_node_stale(upstreams: Iterable[Any]) -> bool: - """Return whether any function or operator job node in ``upstreams`` is stale. + """Return whether any job node in ``upstreams`` that holds run rows is stale. Each such node answers with its own ``is_stale``, which applies this rule to its - own upstreams in turn, so the whole ancestry is covered. Side-effect job nodes are - not followed yet: their own ``is_stale`` is redefined by ITL-1091. + own upstreams in turn, so the whole ancestry is covered. Args: upstreams: The direct upstream streams of a job node. Returns: - ``True`` if any upstream function or operator job node is stale. + ``True`` if any upstream ``RunHoldingNode`` is stale. """ - from orcapod.core.nodes.function_node import FunctionJobNode - from orcapod.core.nodes.operator_node import OperatorJobNode - return any( - isinstance(upstream, (FunctionJobNode, OperatorJobNode)) and upstream.is_stale + isinstance(upstream, RunHoldingNode) and upstream.is_stale for upstream in upstreams ) diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index c38146edc..e71cf1607 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -1396,10 +1396,12 @@ def load( """Deserialize a ``PipelineJob`` from a JSON file to read its stored results. A loaded job is for reading the results its saved run stored; it cannot be - run again, because function pods are not saved with the job. Read a - function node with ``job.nodes[label].as_table()``, ``iter_data()`` or - ``get_all_records()``. Operators can store outputs when caching is enabled; - reading a loaded operator without a readable cache raises ``RuntimeError``. + run again, because function pods are not saved with the job. Its nodes have + not run in this process, so they are empty (ITL-1095): read a node's stored + results through its stored view, ``job.nodes[label].as_source()``, or with + ``get_all_records()``. Operators store outputs only when caching is enabled; + the stored view of a loaded operator whose run's rows cannot be named raises + ``RuntimeError``. The embedded ``Pipeline`` blueprint is always restored. Sources whose data lives in files (CSV, Delta) are rebuilt from the saved config. In-memory @@ -1415,7 +1417,8 @@ def load( over any database configuration in the file. Returns: - A read-only ``PipelineJob`` whose nodes serve their stored results. + A read-only ``PipelineJob`` whose nodes' stored views serve their stored + results. Raises: ValueError: If the file's format version is unsupported, or if it was diff --git a/tests/test_pipeline/loaded_job/test_loaded_job_readback.py b/tests/test_pipeline/loaded_job/test_loaded_job_readback.py index 50a54b71c..b14dff36e 100644 --- a/tests/test_pipeline/loaded_job/test_loaded_job_readback.py +++ b/tests/test_pipeline/loaded_job/test_loaded_job_readback.py @@ -801,8 +801,7 @@ def test_loaded_function_node_cannot_execute(saved: pathlib.Path) -> None: _ = job.nodes["dbl"].execute(job.nodes["dsrc"]) assert str(excinfo.value) == ( "FunctionJobNode 'dbl' was loaded by PipelineJob.load and has no function " - "pod, so it cannot compute; read its stored results with as_table() or " - "iter_data()." + "pod, so it cannot compute; read its stored results with as_source()." ) From 7411a897fd8645a89aab80a52fdfca5059d2c21a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:46:50 -0800 Subject: [PATCH 13/48] test(pipeline): failing tests: job.run(load_only=True) (ITL-1095) Co-Authored-By: Claude Opus 5.5 --- tests/test_pipeline/test_load_only_run.py | 232 ++++++++++++++++++++++ 1 file changed, 232 insertions(+) create mode 100644 tests/test_pipeline/test_load_only_run.py diff --git a/tests/test_pipeline/test_load_only_run.py b/tests/test_pipeline/test_load_only_run.py new file mode 100644 index 000000000..f3c1ff5c9 --- /dev/null +++ b/tests/test_pipeline/test_load_only_run.py @@ -0,0 +1,232 @@ +"""``job.run(load_only=True)``: fill the nodes from what is stored, compute nothing (ITL-1095). + +Developer, November 13, 2026: "it might help for run to have an argument ... load_only=True +where it would fill the pipeline only based on what can be loaded ... and does not trigger +computation of any missing entries." A load-only run reads the sources, runs operators on +the rows that reach them, and fills each function node with the results already stored for +this run's inputs. No user function is called, no side effect is delivered, nothing is +written to the store. Inputs without a stored result are skipped and counted. +""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.side_effects import SideEffectPod, tap_pod +from orcapod.sources import DictSource +from orcapod.types import CacheMode + +CALLS: list[str] = [] +DELIVERED: list[int] = [] + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + """Return ``x + 1``.""" + CALLS.append("inc") + return x + 1 + + +@function_pod(output_keys="total") +def total(y: list[int]) -> int: + """Return the sum of a group's values.""" + CALLS.append("total") + return sum(y) + + +def _deliver(y: int, ctx: Any) -> None: + """A side effect: record the delivered value.""" + DELIVERED.append(y) + + +ROWS = [{"g": i % 2, "k": i, "x": 10 * i} for i in range(6)] +ORCHESTRATORS = {"sync": SyncPipelineOrchestrator, "async": AsyncPipelineOrchestrator} + + +@pytest.fixture(autouse=True) +def _reset() -> None: + CALLS.clear() + DELIVERED.clear() + + +def _build( + data: list[dict[str, int]], + db: Any, + cache_mode: CacheMode | None = None, + with_tap: bool = False, +) -> PipelineJob: + """``rowwise`` (function) -> ``grouped`` (GroupBy) -> ``summed`` (function). + + With ``with_tap``, a side-effect node ``tap`` sits between ``rowwise`` and the group. + """ + src = DictSource(data, tag_columns=["g", "k"], source_id="s") + job = PipelineJob(name="j", database=db) + with job: + rows: Any = inc.pod(src, label="rowwise") + if with_tap: + tap = tap_pod(_deliver) + assert isinstance(tap, SideEffectPod) + rows = tap.process(rows, label="tap") + grouped = rows.group_by("g", label="grouped") + total.pod(grouped, label="summed") + if cache_mode is not None: + job.nodes["grouped"].attach_databases( + pipeline_database=db.at("j"), cache_mode=cache_mode + ) + return job + + +def _snapshot(db: InMemoryArrowDatabase) -> dict[tuple[str, ...], int]: + """Every table of the store with its row count.""" + db.flush() + counts: dict[tuple[str, ...], int] = {} + for path in db.list_table_paths(): + records = db.get_all_records(path) + counts[path] = 0 if records is None else records.num_rows + return counts + + +def _run(job: PipelineJob, orchestrator: str, **kwargs: Any) -> Any: + return job.run(orchestrator=ORCHESTRATORS[orchestrator](), **kwargs) + + +def _totals(job: PipelineJob) -> list[tuple[int, int]]: + return sorted( + (r["g"], r["total"]) for r in job.nodes["summed"].as_table().to_pylist() + ) + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_a_load_only_run_over_a_full_store_equals_a_run_and_computes_nothing( + orchestrator: str, +) -> None: + db = InMemoryArrowDatabase() + _run(_build(ROWS, db), orchestrator) + expected = _build(ROWS, InMemoryArrowDatabase()) + _run(expected, orchestrator) + CALLS.clear() + before = _snapshot(db) + + job = _build(ROWS, db) + summary = _run(job, orchestrator, load_only=True) + + assert CALLS == [] + assert _snapshot(db) == before + assert job.nodes["rowwise"].as_table().num_rows == 6 + assert _totals(job) == _totals(expected) + assert summary.load_only + rowwise = summary.nodes["rowwise"] + assert (rowwise.computed, rowwise.reused, rowwise.not_stored) == (0, 6, 0) + assert rowwise.inputs_incomplete == () + assert summary.nodes["summed"].inputs_incomplete == () + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_inputs_without_a_stored_result_are_skipped_and_counted( + orchestrator: str, +) -> None: + """The store holds a 3-row run; the source now has 6 rows.""" + db = InMemoryArrowDatabase() + _run(_build(ROWS[:3], db), orchestrator) + CALLS.clear() + before = _snapshot(db) + + job = _build(ROWS, db) + summary = _run(job, orchestrator, load_only=True) + + assert CALLS == [] + assert _snapshot(db) == before + assert sorted(r["k"] for r in job.nodes["rowwise"].as_table().to_pylist()) == [ + 0, + 1, + 2, + ] + rowwise = summary.nodes["rowwise"] + assert (rowwise.computed, rowwise.reused, rowwise.not_stored) == (0, 3, 3) + # The groups below hold only the loaded rows: those of the earlier 3-row run, whose + # totals are stored. The summary says their inputs are incomplete. + assert _totals(job) == [(0, 22), (1, 11)] + summed = summary.nodes["summed"] + assert (summed.reused, summed.not_stored) == (2, 0) + assert summed.inputs_incomplete == ("rowwise",) + text = repr(summary) + assert "rowwise: 6 rows: 3 loaded, 3 not stored (skipped)" in text + assert "summed: 2 rows: 2 loaded (inputs incomplete: rowwise)" in text + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_a_load_only_run_over_an_empty_store_fills_nothing_and_writes_nothing( + orchestrator: str, +) -> None: + db = InMemoryArrowDatabase() + job = _build(ROWS, db) + summary = _run(job, orchestrator, load_only=True) + assert CALLS == [] + assert _snapshot(db) == {} + for label in ("rowwise", "grouped", "summed"): + assert job.nodes[label].as_table().num_rows == 0, label + assert summary.nodes["rowwise"].not_stored == 6 + assert summary.nodes["summed"].reused == 0 + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_a_load_only_run_delivers_no_side_effect_and_passes_the_rows_on( + orchestrator: str, +) -> None: + db = InMemoryArrowDatabase() + _run(_build(ROWS, db, with_tap=True), orchestrator) + assert len(DELIVERED) == 6 + DELIVERED.clear() + CALLS.clear() + before = _snapshot(db) + + job = _build(ROWS, db, with_tap=True) + _run(job, orchestrator, load_only=True) + + assert DELIVERED == [] + assert CALLS == [] + assert _snapshot(db) == before + assert _totals(job) == [(0, 63), (1, 93)] + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_a_log_operator_writes_nothing_in_a_load_only_run(orchestrator: str) -> None: + db = InMemoryArrowDatabase() + _run(_build(ROWS[:3], db, cache_mode=CacheMode.LOG), orchestrator) + before = _snapshot(db) + + job = _build(ROWS[:3], db, cache_mode=CacheMode.LOG) + _run(job, orchestrator, load_only=True) + + assert _snapshot(db) == before + assert job.nodes["grouped"].as_table().num_rows == 2 + + +def test_run_async_takes_load_only() -> None: + db = InMemoryArrowDatabase() + _run(_build(ROWS, db), "sync") + CALLS.clear() + job = _build(ROWS, db) + summary = asyncio.run(job.run_async(load_only=True)) + assert CALLS == [] + assert summary.load_only + assert job.nodes["rowwise"].as_table().num_rows == 6 + + +def test_load_only_is_keyword_only() -> None: + job = _build(ROWS, InMemoryArrowDatabase()) + with pytest.raises(TypeError): + job.run(None, None, True, 100, True) # pyright: ignore[reportCallIssue] # load_only is keyword-only + + +def test_a_full_run_summary_is_not_load_only() -> None: + summary = _build(ROWS, InMemoryArrowDatabase()).run() + assert not summary.load_only + assert summary.nodes["rowwise"].not_stored == 0 + assert summary.nodes["rowwise"].inputs_incomplete == () From 6541425ce28ef62244520b4cb60e2d22b6719aac Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:52:16 -0800 Subject: [PATCH 14/48] feat(pipeline): job.run(load_only=True) fills the nodes from what is stored (ITL-1095) job.run, job.run_async and both orchestrators take keyword-only load_only. Sources are read, operators run on the rows that reach them (a LOG cache writes nothing), each function node takes the results stored for this run's inputs (its own records under its content hash) and skips and counts the others, and a side-effect node passes its rows on uncalled. No function call, no delivery, no write. RunSummary.load_only, NodeRunSummary.not_stored and .inputs_incomplete; the text and HTML displays say what was loaded and skipped. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/function_node.py | 174 +++++++++++++++++- src/orcapod/core/nodes/operator_node.py | 15 +- src/orcapod/pipeline/async_orchestrator.py | 48 +++-- src/orcapod/pipeline/job.py | 44 +++-- src/orcapod/pipeline/run_summary.py | 118 +++++++++++- src/orcapod/pipeline/sync_orchestrator.py | 30 ++- src/orcapod/protocols/node_protocols.py | 4 + .../nodes/test_function_node_empty_data.py | 3 +- .../test_job_node_current_run_view.py | 2 +- 9 files changed, 391 insertions(+), 47 deletions(-) diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index cb8a5ced9..e331de106 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -1062,6 +1062,10 @@ class FunctionJobNode(RunHoldingNode, FunctionNodeBase): # its number of inputs; None when no run completed (ITL-1093). A class default so # nodes rebuilt from a saved descriptor start without a run. _run_entry_counts: Counter[bytes] | None = None + # Whether the last completed run was load-only, and how many of its inputs had no + # stored result (ITL-1095). Class defaults for nodes rebuilt from a descriptor. + _run_load_only: bool = False + _run_not_stored: int = 0 def __init__( self, @@ -1526,6 +1530,8 @@ def clear_cache(self) -> None: self._cached_output_table = None self._cached_content_hash_column = None self._run_entry_counts = None + self._run_load_only = False + self._run_not_stored = 0 # No content established: the node, and every node downstream, is stale. self._set_modified_time(None) @@ -1552,12 +1558,162 @@ def _held_pairs(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: if data is not None: yield tag, data - def _set_run_rows(self, entry_ids: list[bytes]) -> None: - """Make the inputs of a completed run, by base entry id, the rows this node holds.""" + def _set_run_rows( + self, entry_ids: list[bytes], *, load_only: bool = False, not_stored: int = 0 + ) -> None: + """Make the inputs of a completed run, by base entry id, the rows this node holds. + + Args: + entry_ids: The base entry ids of the run's inputs whose rows the node holds. + load_only: Whether the run was load-only (ITL-1095). + not_stored: How many inputs of a load-only run had no stored result. + """ self._run_entry_counts = Counter(entry_ids) + self._run_load_only = load_only + self._run_not_stored = not_stored self._cached_output_table = None self._cached_content_hash_column = None + @property + def last_run_not_stored(self) -> int: + """Inputs the node's last completed run skipped because nothing was stored. + + Non-zero only after a load-only run (``run(load_only=True)``); 0 before a run. + """ + return self._run_not_stored + + def _stored_entry( + self, entry_id: bytes + ) -> tuple[TagProtocol, DataProtocol | None] | None: + """The result held for ``entry_id`` after a lookup, or ``None`` if none is stored. + + A placeholder for a result kept in an ephemeral store that is gone is not a + stored result. + """ + entry = self._cached_output_datas.get(entry_id) + if entry is None or isinstance(entry[1], EmptyData): + return None + return entry + + def _execute_load_only( + self, + input_stream: StreamProtocol, + observer: ExecutionObserverProtocol | None, + ) -> list[tuple[TagProtocol, DataProtocol]]: + """Fill the node with the stored results of ``input_stream``'s rows; compute nothing. + + The load-only form of ``execute`` (ITL-1095): each input's entry is looked up + among the node's own stored entries (its pipeline records under its content + hash, the lookup a run makes before computing). An input with a stored result + is a loaded row (``on_data_start`` and ``on_data_end(cached=True)``); one + without is skipped and counted (``last_run_not_stored``), with no observer + call. No function is called and nothing is written. + + Returns: + The loaded (tag, output data) rows, excluding ``None`` outputs. + + Raises: + RuntimeError: If no store is attached (``UNAVAILABLE``). + InputValidationError: If the input stream is of another data context. + """ + from orcapod.pipeline.observer import NoOpObserver + from orcapod.pipeline.serialization import LoadStatus + + if self.load_status == LoadStatus.UNAVAILABLE: + raise self._no_store_error() + require_same_data_context( + input_stream.data_context_key, self._stored_data_context_key + ) + node_label = self.label + node_hash = self.content_hash().to_string() + obs = observer if observer is not None else NoOpObserver() + ctx_obs = obs.contextualize(*self.node_identity_path) + tag_schema = input_stream.output_schema(columns={"system_tags": True})[0] + ctx_obs.on_node_start(node_label, node_hash, tag_schema=tag_schema) + output: list[tuple[TagProtocol, DataProtocol]] = [] + loaded_ids: list[bytes] = [] + not_stored = 0 + try: + inputs = [ + (tag, data, self.compute_base_entry_id(tag, data)) + for tag, data in input_stream.iter_data() + ] + _ = self.get_cached_results(base_entry_ids=[eid for _, _, eid in inputs]) + for tag, data, entry_id in inputs: + entry = self._stored_entry(entry_id) + if entry is None: + not_stored += 1 + continue + ctx_obs.on_data_start(node_label, tag, data) + ctx_obs.on_data_end(node_label, tag, data, entry[1], cached=True) + loaded_ids.append(entry_id) + if entry[1] is not None: + output.append((entry[0], entry[1])) + except BaseException: + # A run that raised defines no rows. + self.clear_cache() + raise + self._set_run_rows(loaded_ids, load_only=True, not_stored=not_stored) + ctx_obs.on_node_end(node_label, node_hash) + self._update_modified_time() + return output + + async def _async_execute_load_only( + self, + input_channel: ReadableChannel[tuple[TagProtocol, DataProtocol]], + output: WritableChannel[tuple[TagProtocol, DataProtocol]], + observer: ExecutionObserverProtocol | None, + ) -> None: + """The async form of ``_execute_load_only``: same rows, counts and observer calls. + + The node reads its whole input before it looks the entries up, then sends the + loaded rows; it stamps itself before its output closes (ITL-1042), and closes + the output whatever happens. + """ + from orcapod.pipeline.observer import NoOpObserver + from orcapod.pipeline.serialization import LoadStatus + + try: + if self.load_status == LoadStatus.UNAVAILABLE: + raise self._no_store_error() + require_same_data_context( + self._input_stream.data_context_key, self._stored_data_context_key + ) + node_label = self.label + node_hash = self.content_hash().to_string() + obs = observer if observer is not None else NoOpObserver() + ctx_obs = obs.contextualize(*self.node_identity_path) + tag_schema = self._input_stream.output_schema( + columns={"system_tags": True} + )[0] + ctx_obs.on_node_start(node_label, node_hash, tag_schema=tag_schema) + inputs = [ + (tag, data, self.compute_base_entry_id(tag, data)) + async for tag, data in input_channel + ] + _ = self.get_cached_results(base_entry_ids=[eid for _, _, eid in inputs]) + loaded_ids: list[bytes] = [] + not_stored = 0 + for tag, data, entry_id in inputs: + entry = self._stored_entry(entry_id) + if entry is None: + not_stored += 1 + continue + ctx_obs.on_data_start(node_label, tag, data) + ctx_obs.on_data_end(node_label, tag, data, entry[1], cached=True) + loaded_ids.append(entry_id) + if entry[1] is not None: + await output.send((entry[0], entry[1])) + self._set_run_rows(loaded_ids, load_only=True, not_stored=not_stored) + ctx_obs.on_node_end(node_label, node_hash) + self._update_modified_time() + except BaseException: + # A run that raised or was cancelled defines no rows. + self.clear_cache() + raise + finally: + await output.close() + def set_ephemeral_store(self, store: "ArrowDatabaseProtocol | None") -> None: """Assign or remove the ephemeral result store. @@ -1912,6 +2068,7 @@ def execute( observer: ExecutionObserverProtocol | None = None, error_policy: Literal["continue", "fail_fast"] = "continue", run_id: str | None = None, + load_only: bool = False, ) -> list[tuple[TagProtocol, DataProtocol]]: """Execute all data from a stream: compute, persist, and cache. @@ -1931,6 +2088,11 @@ def execute( ``"fail_fast"`` re-raises on the first failure. run_id: Optional pipeline run identifier threaded through to ``InvocationContext.pipeline_run_id``. + load_only: Fill the node with the results already stored for the + stream's rows and compute nothing (ITL-1095): no function call, no + write; inputs without a stored result are skipped and counted + (``last_run_not_stored``). A loaded node (no function pod) can run + this way. Returns: Materialized list of (tag, output_data) pairs, excluding @@ -1945,6 +2107,8 @@ def execute( """ from orcapod.pipeline.observer import NoOpObserver + if load_only: + return self._execute_load_only(input_stream, observer) if self._function_pod is None: raise self._cannot_compute_error() @@ -3274,6 +3438,7 @@ async def async_execute( *, observer: ExecutionObserverProtocol | None = None, run_id: str | None = None, + load_only: bool = False, ) -> None: """Streaming async execution for FunctionJobNode. @@ -3296,6 +3461,8 @@ async def async_execute( observer: Optional execution observer for hooks. run_id: Optional pipeline run identifier threaded through to ``InvocationContext.pipeline_run_id``. + load_only: As for ``execute``: fill the node with the stored results of + its input rows and compute nothing (ITL-1095). Raises: RuntimeError: If the node was loaded by ``PipelineJob.load``: without a @@ -3304,6 +3471,9 @@ async def async_execute( """ from orcapod.pipeline.serialization import LoadStatus + if load_only: + await self._async_execute_load_only(input_channel, output, observer) + return try: self._begin_run() except StoreError: diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 7ca6367d4..16c3321d6 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -1322,12 +1322,17 @@ def execute( self, *input_streams: StreamProtocol, observer: ExecutionObserverProtocol | None = None, + load_only: bool = False, ) -> list[tuple[TagProtocol, DataProtocol]]: """Execute input streams: compute, persist, and cache. Args: *input_streams: Input streams to execute. observer: Optional execution observer for hooks. + load_only: A load-only run (ITL-1095): the operator still runs on the rows + that reach it (it is not a stored computation), but nothing is + written: a ``LOG`` node does not store its output. ``REPLAY`` replays + as in any run. Returns: Materialized list of (tag, data) pairs. @@ -1368,9 +1373,10 @@ def execute( self._update_modified_time() - # Persist to DB only in LOG mode + # Persist to DB only in LOG mode, and never in a load-only run. if ( - self._pipeline_database is not None + not load_only + and self._pipeline_database is not None and self._cache_mode == CacheMode.LOG and self._cached_output_stream is not None ): @@ -1667,6 +1673,7 @@ async def async_execute( output: WritableChannel[tuple[TagProtocol, DataProtocol]], *, observer: "ExecutionObserverProtocol | None" = None, + load_only: bool = False, ) -> None: """Async execution with cache mode handling when DB is attached. @@ -1688,6 +1695,7 @@ async def async_execute( inputs: Sequence of readable channels from upstream nodes. output: Writable channel for output (tag, data) pairs. observer: Optional execution observer for hooks. + load_only: As for ``execute``: a ``LOG`` node stores nothing (ITL-1095). """ from orcapod.pipeline.observer import NoOpObserver @@ -1733,7 +1741,8 @@ async def async_execute( # Delegate to the operator, forward its rows downstream and keep them. intermediate: Channel[tuple[TagProtocol, DataProtocol]] = Channel() should_store = ( - self._pipeline_database is not None + not load_only + and self._pipeline_database is not None and self._cache_mode == CacheMode.LOG ) if should_store: diff --git a/src/orcapod/pipeline/async_orchestrator.py b/src/orcapod/pipeline/async_orchestrator.py index 07f53b7f7..592d6fdb7 100644 --- a/src/orcapod/pipeline/async_orchestrator.py +++ b/src/orcapod/pipeline/async_orchestrator.py @@ -31,6 +31,7 @@ ) if TYPE_CHECKING: + from orcapod.channels import WritableChannel from orcapod.pipeline.dag import GraphProtocol from orcapod.protocols.core_protocols import DataProtocol, TagProtocol from orcapod.protocols.observability_protocols import ExecutionObserverProtocol @@ -75,6 +76,7 @@ def run( materialize_results: bool = True, run_id: str | None = None, pipeline_uri: str = "", + load_only: bool = False, ) -> OrchestratorResult: """Synchronous entry point — runs the async pipeline to completion. @@ -118,6 +120,7 @@ def run( observer=observer, run_id=run_id, pipeline_uri=pipeline_uri, + load_only=load_only, ) ) @@ -138,6 +141,7 @@ def make_run() -> Coroutine[Any, Any, OrchestratorResult]: observer=watched, run_id=run_id, pipeline_uri=pipeline_uri, + load_only=load_only, ) return _run_on_own_thread(make_run, rows_in_flight) @@ -150,6 +154,7 @@ async def run_async( materialize_results: bool = True, run_id: str | None = None, pipeline_uri: str = "", + load_only: bool = False, ) -> OrchestratorResult: """Async entry point for callers already inside an event loop. @@ -171,6 +176,7 @@ async def run_async( observer=observer, run_id=run_id, pipeline_uri=pipeline_uri, + load_only=load_only, ) async def _run_async( @@ -181,6 +187,7 @@ async def _run_async( observer: ExecutionObserverProtocol | None = None, run_id: str | None = None, pipeline_uri: str = "", + load_only: bool = False, ) -> OrchestratorResult: """Core async logic: wire channels, launch tasks, collect results.""" from orcapod.pipeline.observer import NoOpObserver @@ -261,6 +268,7 @@ async def _run_async( writer, observer=effective_observer, run_id=run_id, + load_only=load_only, ), ) elif is_operator_node(node): @@ -280,7 +288,10 @@ async def _run_async( tg, node, node.async_execute( - input_readers, writer, observer=effective_observer + input_readers, + writer, + observer=effective_observer, + load_only=load_only, ), ) elif is_side_effect_node(node): @@ -290,16 +301,20 @@ async def _run_async( "SideEffectNode expects exactly 1 upstream, got 0" ) input_reader = edge_readers[(predecessors[0], node)] - _start_node( - tg, - node, - node.async_execute( - [input_reader], - writer, - observer=effective_observer, - run_id=run_id, - ), - ) + if load_only: + # Nothing is delivered in a load-only run: rows pass on. + _start_node(tg, node, _pass_through(input_reader, writer)) + else: + _start_node( + tg, + node, + node.async_execute( + [input_reader], + writer, + observer=effective_observer, + run_id=run_id, + ), + ) else: raise TypeError( f"Unknown node type: {getattr(node, 'node_type', None)!r}" @@ -595,6 +610,17 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: return NoOpLogger() +async def _pass_through( + reader: Any, writer: WritableChannel[tuple[TagProtocol, DataProtocol]] +) -> None: + """Forward every row of ``reader`` to ``writer``, then close ``writer``.""" + try: + async for item in reader: + await writer.send(item) + finally: + await writer.close() + + def _start_node( tg: asyncio.TaskGroup, node: Any, work: Coroutine[Any, Any, None] ) -> None: diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index e71cf1607..eee77a79a 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -31,13 +31,9 @@ from orcapod.pipeline.base import AbstractPipelineBase from orcapod.pipeline.dag import OrcaDAG from orcapod.pipeline.run_summary import ( - NodeRunSummary, RunCountingObserver, RunSummary, -) -from orcapod.protocols.node_protocols import ( - is_function_node, - is_side_effect_function_node, + run_node_summaries, ) from orcapod.protocols import core_protocols as cp from orcapod.databases.utils import as_table_committing @@ -866,6 +862,8 @@ def run( observer: "ExecutionObserverProtocol | None" = None, materialize_results: bool = True, commit_every_rows: int = 100, + *, + load_only: bool = False, ) -> RunSummary: """Execute the resolvable subgraph of this job in place. @@ -914,6 +912,15 @@ def run( default 1000): the store commits every pending table by itself once one table's pending rows reach that size, which would break the node's results-before-records commit order. + load_only: Fill the nodes from what is already stored and compute + nothing (ITL-1095). Sources are read and operators run on the rows + that reach them; each function node takes the results stored for + this run's inputs (its own records under its content hash) and skips + the inputs without one, which the summary counts (``not_stored``); + a function node forwards only the rows it loaded. No user function + is called, no side effect is delivered (a side-effect node passes its + rows on) and nothing is written to the store, a ``LOG`` operator's + cache included. Afterwards each node shows the rows it loaded. Rows whose function raised (with the default ``error_policy="continue"``) are skipped, and the run goes on. At the end, if any row failed, a single @@ -956,8 +963,9 @@ def run( run_id=prepared.run_id, pipeline_uri=prepared.pipeline_uri, materialize_results=materialize_results, + load_only=load_only, ) - summary = self._finish_run(prepared, counter) + summary = self._finish_run(prepared, counter, load_only=load_only) if not summary.ok: _warn_row_failures(summary, sys._getframe(1)) # pyright: ignore[reportPrivateUsage] # the caller's line is where the warning is shown return summary @@ -968,6 +976,8 @@ async def run_async( observer: "ExecutionObserverProtocol | None" = None, materialize_results: bool = True, commit_every_rows: int = 100, + *, + load_only: bool = False, ) -> RunSummary: """Execute the job with the async orchestrator, from async code. @@ -983,6 +993,7 @@ async def run_async( observer: Optional execution observer. materialize_results: As for ``run``. commit_every_rows: As for ``run``. + load_only: As for ``run``. Returns: The run's ``RunSummary``, as ``run`` returns. @@ -1014,8 +1025,9 @@ async def run_async( run_id=prepared.run_id, pipeline_uri=prepared.pipeline_uri, materialize_results=materialize_results, + load_only=load_only, ) - summary = self._finish_run(prepared, counter) + summary = self._finish_run(prepared, counter, load_only=load_only) if not summary.ok: _warn_row_failures(summary, sys._getframe(1)) # pyright: ignore[reportPrivateUsage] # the caller's line is where the warning is shown return summary @@ -1215,13 +1227,18 @@ def _executing( raise def _finish_run( - self, prepared: "_PreparedRun", counter: RunCountingObserver + self, + prepared: "_PreparedRun", + counter: RunCountingObserver, + *, + load_only: bool = False, ) -> RunSummary: """Commit the run's last writes, mark the job as run and keep its summary. Args: prepared: The run, from ``_prepare_run``. counter: The observer that counted the run's rows. + load_only: Whether the run loaded stored results only. Returns: The run's summary, also kept as ``last_run``. @@ -1255,12 +1272,11 @@ def _finish_run( self._has_run = True self._last_run_finished = True self._run_id = prepared.run_id - node_summaries: list[NodeRunSummary] = [] - for node in prepared.exec_dag.topological_sort(): - label: str = node.label - if is_function_node(node) or is_side_effect_function_node(node): - node_summaries.append(counter.node_summary(label)) - self._last_run = RunSummary("/".join(self._name), node_summaries) + self._last_run = RunSummary( + "/".join(self._name), + run_node_summaries(prepared.exec_dag, counter), + load_only=load_only, + ) return self._last_run def __repr__(self) -> str: diff --git a/src/orcapod/pipeline/run_summary.py b/src/orcapod/pipeline/run_summary.py index cf094c1c4..84124357d 100644 --- a/src/orcapod/pipeline/run_summary.py +++ b/src/orcapod/pipeline/run_summary.py @@ -37,6 +37,11 @@ class NodeRunSummary: failed: Rows whose function raised; the run skipped them. first_error: The exception of the first failed row, with its traceback, or ``None`` when no row failed. + not_stored: Inputs a load-only run skipped because no result is stored for + them (``job.run(load_only=True)``; always 0 in a full run). + inputs_incomplete: The labels of the function nodes upstream of this one that + skipped inputs in a load-only run, in run order: this node's inputs lack + the rows those results would have given (a group may be partial). """ label: str @@ -44,11 +49,13 @@ class NodeRunSummary: reused: int failed: int first_error: BaseException | None + not_stored: int = 0 + inputs_incomplete: tuple[str, ...] = () @property def rows(self) -> int: - """All rows the node received: computed, reused and failed.""" - return self.computed + self.reused + self.failed + """All rows the node received: computed, reused, failed and not stored.""" + return self.computed + self.reused + self.failed + self.not_stored class RunSummary: @@ -61,17 +68,27 @@ class RunSummary: Attributes: job_name: The job's name. nodes: Per function node, in run order, keyed by label. + load_only: Whether the run was a load-only run (``job.run(load_only=True)``): + it computed nothing; ``reused`` counts the rows it loaded. """ - def __init__(self, job_name: str, nodes: Sequence[NodeRunSummary]) -> None: + def __init__( + self, + job_name: str, + nodes: Sequence[NodeRunSummary], + *, + load_only: bool = False, + ) -> None: """Build the summary. Args: job_name: The job's name. nodes: One entry per function node of the run, in run order. + load_only: Whether the run loaded stored results only. """ self.job_name: str = job_name self.nodes: Mapping[str, NodeRunSummary] = {node.label: node for node in nodes} + self.load_only: bool = load_only @property def failed_rows(self) -> int: @@ -121,6 +138,14 @@ def print_traceback(self, label: str | None = None) -> None: def _headline(self) -> str: """The first line: the job and whether any row failed.""" + if self.load_only: + skipped = sum(node.not_stored for node in self.nodes.values()) + outcome = ( + f"{_rows(skipped)} without a stored result skipped." + if skipped + else "every input had a stored result." + ) + return f"Load-only run of job {self.job_name!r}: {outcome}" if self.failed_rows: outcome = f"{_rows(self.failed_rows)} failed." else: @@ -132,8 +157,7 @@ def __repr__(self) -> str: lines = [self._headline()] for node in self.nodes.values(): lines.append( - f" {node.label}: {_rows(node.rows)}: {node.computed} computed, " - f"{node.reused} from store, {node.failed} failed" + f" {node.label}: {_rows(node.rows)}: {_counts(node, self.load_only)}" ) if node.failed: lines.append(f" first error: {_error_line(node.first_error)}") @@ -142,6 +166,8 @@ def __repr__(self) -> str: def _repr_html_(self) -> str: """The notebook display: the same content as ``repr``, as a table.""" + if self.load_only: + return self._load_only_html() header = "".join( f"{name}" for name in ( @@ -177,6 +203,40 @@ def _repr_html_(self) -> str: f"{''.join(body)}" ) + def _load_only_html(self) -> str: + """The notebook display of a load-only run: rows loaded and skipped per node.""" + header = "".join( + f"{name}" + for name in ("node", "rows", "loaded", "not stored", "inputs incomplete") + ) + body = [ + "" + f"{html.escape(node.label)}" + f"{node.rows}{node.reused}{node.not_stored}" + f"{html.escape(', '.join(node.inputs_incomplete))}" + "" + for node in self.nodes.values() + ] + return ( + f"

{html.escape(self._headline())}

" + f"{header}" + f"{''.join(body)}
" + ) + + +def _counts(node: NodeRunSummary, load_only: bool) -> str: + """A node's counts as text: what a full run did, or what a load-only run loaded.""" + if not load_only: + return ( + f"{node.computed} computed, {node.reused} from store, {node.failed} failed" + ) + text = f"{node.reused} loaded" + if node.not_stored: + text += f", {node.not_stored} not stored (skipped)" + if node.inputs_incomplete: + text += f" (inputs incomplete: {', '.join(node.inputs_incomplete)})" + return text + def _rows(n: int) -> str: """``1 row`` or ``n rows``.""" @@ -289,3 +349,51 @@ def on_data_crash( def create_data_logger(self, tag: TagProtocol, data: DataProtocol) -> Any: """No logging of its own.""" return _NOOP_LOGGER + + +def run_node_summaries(dag: Any, counter: RunCountingObserver) -> list[NodeRunSummary]: + """One summary per function node of a finished run, in run order. + + The counts come from ``counter``; a load-only run's skipped inputs come from each + node (``last_run_not_stored``), and every node below a function node that skipped + inputs names it in ``inputs_incomplete`` (ITL-1095). + + Args: + dag: The run's node graph (``topological_sort``, ``predecessors``). + counter: The observer that counted the run's rows. + + Returns: + The summaries of the run's function nodes, in run order. + """ + from dataclasses import replace + + from orcapod.protocols.node_protocols import ( + is_function_node, + is_side_effect_function_node, + ) + + order = list(dag.topological_sort()) + position = {id(node): index for index, node in enumerate(order)} + skipping_above: dict[int, set[int]] = {} + summaries: list[NodeRunSummary] = [] + for node in order: + above: set[int] = set() + for upstream in dag.predecessors(node): + above |= skipping_above.get(id(upstream), set()) + if getattr(upstream, "last_run_not_stored", 0): + above.add(id(upstream)) + skipping_above[id(node)] = above + if not (is_function_node(node) or is_side_effect_function_node(node)): + continue + labels = tuple( + order[index].label for index in sorted(position[key] for key in above) + ) + label: str = getattr(node, "label") # every graph node is labelled + summaries.append( + replace( + counter.node_summary(label), + not_stored=getattr(node, "last_run_not_stored", 0), + inputs_incomplete=labels, + ) + ) + return summaries diff --git a/src/orcapod/pipeline/sync_orchestrator.py b/src/orcapod/pipeline/sync_orchestrator.py index 981706352..af57145b3 100644 --- a/src/orcapod/pipeline/sync_orchestrator.py +++ b/src/orcapod/pipeline/sync_orchestrator.py @@ -55,6 +55,7 @@ def run( materialize_results: bool = True, run_id: str | None = None, pipeline_uri: str = "", + load_only: bool = False, ) -> OrchestratorResult: """Execute the node graph synchronously. @@ -69,6 +70,10 @@ def run( pipeline_uri: Opaque URI string identifying this pipeline run (e.g. ``"my_pipeline@a1b2c3d4e5f6a1b2"``). Forwarded verbatim to ``observer.on_run_start``. + load_only: Fill the nodes from what is stored and compute nothing + (ITL-1095): function nodes load their inputs' stored results and skip + the others, operators run without storing, and a side-effect node + passes its rows on without being called (nothing is delivered). Returns: OrchestratorResult with node outputs. @@ -101,6 +106,7 @@ def run( observer=effective_observer, error_policy=self._error_policy, run_id=run_id, + load_only=load_only, ) elif is_operator_node(node): upstream_buffers = self._gather_upstream_multi(node, graph, buffers) @@ -109,19 +115,23 @@ def run( for buf, upstream_node in upstream_buffers ] buffers[node] = node.execute( - *input_streams, observer=effective_observer + *input_streams, observer=effective_observer, load_only=load_only ) elif is_side_effect_node(node): upstream_buf = self._gather_upstream(node, graph, buffers) - upstream_node = list(graph.predecessors(node))[0] - input_stream = self._materialize_as_stream( - upstream_buf, upstream_node - ) - buffers[node] = node.execute( - input_stream, - observer=effective_observer, - run_id=run_id, - ) + if load_only: + # Nothing is delivered in a load-only run: the rows pass on. + buffers[node] = list(upstream_buf) + else: + upstream_node = list(graph.predecessors(node))[0] + input_stream = self._materialize_as_stream( + upstream_buf, upstream_node + ) + buffers[node] = node.execute( + input_stream, + observer=effective_observer, + run_id=run_id, + ) else: raise TypeError( f"Unknown node type: {getattr(node, 'node_type', None)!r}" diff --git a/src/orcapod/protocols/node_protocols.py b/src/orcapod/protocols/node_protocols.py index 98894e2e8..c33a71c84 100644 --- a/src/orcapod/protocols/node_protocols.py +++ b/src/orcapod/protocols/node_protocols.py @@ -88,6 +88,7 @@ def execute( observer: ExecutionObserverProtocol | None = None, error_policy: Literal["continue", "fail_fast"] = "continue", run_id: str | None = None, + load_only: bool = False, ) -> list[tuple[TagProtocol, DataProtocol]]: ... async def async_execute( @@ -97,6 +98,7 @@ async def async_execute( *, observer: ExecutionObserverProtocol | None = None, run_id: str | None = None, + load_only: bool = False, ) -> None: ... def set_ephemeral_store(self, store: "ArrowDatabaseProtocol | None") -> None: @@ -125,6 +127,7 @@ def execute( self, *input_streams: StreamProtocol, observer: ExecutionObserverProtocol | None = None, + load_only: bool = False, ) -> list[tuple[TagProtocol, DataProtocol]]: ... async def async_execute( @@ -133,6 +136,7 @@ async def async_execute( output: WritableChannel[tuple[TagProtocol, DataProtocol]], *, observer: ExecutionObserverProtocol | None = None, + load_only: bool = False, ) -> None: ... def set_ephemeral_store(self, store: "ArrowDatabaseProtocol | None") -> None: diff --git a/tests/test_core/nodes/test_function_node_empty_data.py b/tests/test_core/nodes/test_function_node_empty_data.py index 7edf9222d..b6ca7555b 100644 --- a/tests/test_core/nodes/test_function_node_empty_data.py +++ b/tests/test_core/nodes/test_function_node_empty_data.py @@ -13,6 +13,7 @@ from orcapod.core.nodes.function_node import FunctionJobNode from orcapod.core.sources import ArrowTableSource from orcapod.databases import InMemoryArrowDatabase +from orcapod.protocols.core_protocols import DataProtocol, TagProtocol from orcapod.system_constants import constants from orcapod.types import NodeConfig from orcapod.errors import EphemeralResultMissingError @@ -174,7 +175,7 @@ def test_normal_result_wins_over_empty_data_token(self, persistent_node): assert isinstance(data, Data) -def _entries(node: FunctionJobNode) -> list: +def _entries(node: FunctionJobNode) -> list[tuple[TagProtocol, DataProtocol]]: """The node's entries for its input rows, as its cache façade returns them. These tests drive ``execute_data`` (one row, not a run), so the node shows no rows diff --git a/tests/test_pipeline/test_job_node_current_run_view.py b/tests/test_pipeline/test_job_node_current_run_view.py index 5b6cf24cd..e9ad8c4b1 100644 --- a/tests/test_pipeline/test_job_node_current_run_view.py +++ b/tests/test_pipeline/test_job_node_current_run_view.py @@ -59,7 +59,7 @@ def _build( src = DictSource(data, tag_columns=["g", "k"], source_id="s") job = PipelineJob(name="j", database=db) with job: - rowwise = inc.pod(src, label="rowwise") + rowwise: Any = inc.pod(src, label="rowwise") grouped = rowwise.group_by("g", label="grouped") total.pod(grouped, label="summed") if cache_mode is not None: From c90eb96662487df4f6ddcee52a55a35aa95a71a8 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:52:55 -0800 Subject: [PATCH 15/48] docs: loading without computing, job.run(load_only=True) (ITL-1095) Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 4 +++- CLAUDE.md | 4 +++- changelog.d/ITL-1095.md | 4 ++++ docs/concepts/function-pods.md | 36 ++++++++++++++++++++++++++++++++++ 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/.zed/rules b/.zed/rules index 6653ea945..f77ed3ea4 100644 --- a/.zed/rules +++ b/.zed/rules @@ -783,7 +783,9 @@ data. The rules: when any function or operator job node it depends on is (core/nodes/staleness.py), and a SourceJobNode is modified when it is bound or its bound source is. A run skips inputs whose entry is already stored. A loaded operator's stored view raises when its run's rows - cannot be named (ITL-1043). + cannot be named (ITL-1043). job.run(load_only=True) computes nothing and writes nothing: function nodes + load their own stored entries for this run's inputs and skip (and count) the others, + operators run without storing, side-effect nodes pass rows on undelivered. - Empty data → ArrowTableSource raises ValueError("Table is empty"). - DerivedSource over a job node with nothing stored → presents zero rows with the origin's columns (no computed records; no error), in the plain form and with tags_as_data. diff --git a/CLAUDE.md b/CLAUDE.md index 66a284382..93ff72c09 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -824,7 +824,9 @@ data. The rules: when any function or operator job node it depends on is (`core/nodes/staleness.py`), and a `SourceJobNode` is modified when it is bound or its bound source is. A run skips inputs whose entry is already stored. A loaded operator's stored view raises when its run's rows - cannot be named (ITL-1043). + cannot be named (ITL-1043). `job.run(load_only=True)` computes nothing and writes nothing: function nodes + load their own stored entries for this run's inputs and skip (and count) the others, + operators run without storing, side-effect nodes pass rows on undelivered. - Empty data → `ArrowTableSource` raises `ValueError("Table is empty")`. - `DerivedSource` over a job node with nothing stored → presents zero rows with the origin's columns (no computed records; no error), in the plain form and with `tags_as_data`. diff --git a/changelog.d/ITL-1095.md b/changelog.d/ITL-1095.md index 0a5c89eeb..8577bb28c 100644 --- a/changelog.d/ITL-1095.md +++ b/changelog.d/ITL-1095.md @@ -4,6 +4,10 @@ - A function or operator job node that has not run in this process is stale (`is_stale` is `True`; it has no modified time), and a job node is stale also when any function or operator job node it depends on is. Binding a source to a job's source slot (`job.bind(sources=...)`) marks the slot modified, so every node fed by it is stale and empty until it runs again. - A loaded operator node no longer raises from `as_table()` or `iter_data()`: it is empty, like every node that has not run. Its stored view (`as_source()`, `get_all_records()`) raises instead when the job file does not record its run's content hash and the store holds nothing under the hash the loaded job computes (ITL-1043). +### Added + +- `job.run(load_only=True)` and `await job.run_async(load_only=True)` fill a job's nodes with what is already stored for this run's inputs and compute nothing (ITL-1095): sources are read, operators run on the rows that reach them (a `CacheMode.LOG` cache writes nothing), each function node takes the stored results of its inputs and skips the inputs without one, a side-effect node passes its rows on without delivering anything, and nothing is written to the store. `RunSummary.load_only`, `NodeRunSummary.not_stored` (inputs skipped) and `NodeRunSummary.inputs_incomplete` (the function nodes above a node that skipped inputs) report it; loaded rows count in `reused`, and the summary's text says "3 loaded, 1 not stored (skipped)". + ### Fixed - On the asynchronous orchestrator, an operator node in `CacheMode.REPLAY` stamps itself after its inputs close, so the rows it replayed are not judged out of date by the time the run ends. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index d32956901..c01237a4f 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -541,6 +541,42 @@ print(summary.ok, summary.failed_rows) # False 3 ``` +### Loading without computing: `run(load_only=True)` + +A job node shows the rows of its current run, so a job built over a store that already holds +its results is empty until it runs. A run that finds every result stored computes nothing, +but it does compute the results that are missing. `job.run(load_only=True)` fills the nodes +with what is already stored for this run's inputs and never computes: sources are read, +operators run on the rows that reach them, each function node takes the stored results of +its inputs and skips the inputs that have none, no side effect is delivered and nothing is +written to the store. The summary counts what each node loaded and skipped: + +```python +more_mice = mice + [{"subject_id": "mouse_04", "weight": 30.0, "height": 0.12}] + +load_job = PipelineJob(name="bmi_demo", database=db) +with load_job: + source = DictSource(data=more_mice, tag_columns=["subject_id"], source_id="mice") + compute_bmi.pod(source, label="bmi") + +print(load_job.nodes["bmi"].as_table().num_rows) # not run in this process: empty +# 0 +calls.clear() +summary = load_job.run(load_only=True) +print(summary) +# Load-only run of job 'bmi_demo': 1 row without a stored result skipped. +# bmi: 4 rows: 3 loaded, 1 not stored (skipped) +print(load_job.nodes["bmi"].as_table().num_rows, "rows; function calls:", len(calls)) +# 3 rows; function calls: 0 +``` + +`summary.nodes["bmi"].not_stored` is the number of skipped inputs, and +`summary.nodes[label].inputs_incomplete` names the function nodes above a node that skipped +inputs. A function node passes on only the rows it loaded, so the rows below it are +incomplete: a group formed after it may lack members, and the result stored for a smaller +group by an earlier run may then be loaded for it. `run_async(load_only=True)` does the same +on the asynchronous orchestrator. + ### When results are committed Inside `job.run()`, each function node commits its results and pipeline records to the From d389828275f8e566ec0af8a6b9fb4b28efc9f0e0 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:56:50 -0800 Subject: [PATCH 16/48] test(pipeline): failing tests: node.run() cascade, state rule and load-only status (ITL-1095) Co-Authored-By: Claude Opus 5.5 --- tests/test_pipeline/test_node_run_cascade.py | 314 +++++++++++++++++++ 1 file changed, 314 insertions(+) create mode 100644 tests/test_pipeline/test_node_run_cascade.py diff --git a/tests/test_pipeline/test_node_run_cascade.py b/tests/test_pipeline/test_node_run_cascade.py new file mode 100644 index 000000000..f986068ab --- /dev/null +++ b/tests/test_pipeline/test_node_run_cascade.py @@ -0,0 +1,314 @@ +"""``node.run()`` runs the job nodes it depends on, by their state (ITL-1095). + +Developer, November 13, 2026: "given that the initially loaded job is empty, I think it's +fair that node.run will cascade node.run on all nodes it depends on recursively. +Critically, node.run(load_only=True) would trigger the upstream node.run also with +load_only=True." And: "if it's not stale, it shouldn't even trigger a run and simply return +what's in the stream cached already." A node filled by a load-only run has its own status +and is never treated as complete by a full run. + +The cascade runs the node's ancestors and the node on the synchronous orchestrator, as +``job.run()`` runs that sub-graph: same rows, same stored records, same counts +(``node.last_run``), and the same end-of-run commit. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any +from unittest import mock + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.core.operators import Join +from orcapod.core.streams import ViewStatus +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.errors import RowFailuresWarning +from orcapod.side_effects import SideEffectPod, tap_pod +from orcapod.sources import DictSource +from orcapod.types import CacheMode + +CALLS: list[str] = [] +DELIVERED: list[int] = [] + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + """Return ``x + 1``.""" + CALLS.append("inc") + return x + 1 + + +@function_pod(output_keys="total") +def total(y: list[int]) -> int: + """Return the sum of a group's values.""" + CALLS.append("total") + return sum(y) + + +@function_pod(output_keys="z") +def dbl(y: int) -> int: + """Return ``2 * y``.""" + CALLS.append("dbl") + return 2 * y + + +@function_pod(output_keys="w") +def neg(y: int) -> int: + """Return ``-y``.""" + CALLS.append("neg") + return -y + + +@function_pod(output_keys="y") +def fails_on_20(x: int) -> int: + """Return ``x + 1``; raise for ``x == 20``.""" + if x == 20: + raise ValueError("x is 20") + return x + 1 + + +def _deliver(y: int, ctx: Any) -> None: + """A side effect: record the delivered value.""" + DELIVERED.append(y) + + +ROWS = [{"g": i % 2, "k": i, "x": 10 * i} for i in range(6)] + + +@pytest.fixture(autouse=True) +def _reset() -> None: + CALLS.clear() + DELIVERED.clear() + + +def _source(data: list[dict[str, int]]) -> DictSource: + return DictSource(data, tag_columns=["g", "k"], source_id="s") + + +def _build( + data: list[dict[str, int]], + db: Any, + *, + cache_mode: CacheMode | None = None, + with_tap: bool = False, +) -> PipelineJob: + """``rowwise`` -> [``tap``] -> ``grouped`` (GroupBy) -> ``summed``.""" + job = PipelineJob(name="j", database=db) + with job: + rows: Any = inc.pod(_source(data), label="rowwise") + if with_tap: + tap = tap_pod(_deliver) + assert isinstance(tap, SideEffectPod) + rows = tap.process(rows, label="tap") + grouped = rows.group_by("g", label="grouped") + total.pod(grouped, label="summed") + if cache_mode is not None: + job.nodes["grouped"].attach_databases( + pipeline_database=db.at("j"), cache_mode=cache_mode + ) + return job + + +def _snapshot(db: Any) -> dict[tuple[str, ...], int]: + db.flush() + counts: dict[tuple[str, ...], int] = {} + for path in db.list_table_paths(): + records = db.get_all_records(path) + counts[path] = 0 if records is None else records.num_rows + return counts + + +def _totals(table: Any) -> list[tuple[int, int]]: + return sorted((r["g"], r["total"]) for r in table.to_pylist()) + + +class TestCascade: + """``node.run()`` first runs every job node it depends on, each once.""" + + def test_a_node_of_a_fresh_job_runs_its_inputs_first(self) -> None: + job = _build(ROWS, InMemoryArrowDatabase()) + view = job.nodes["summed"].run() + assert sorted(CALLS) == ["inc"] * 6 + ["total"] * 2 + assert _totals(view.as_table()) == [(0, 63), (1, 93)] + # Every node of the cascade holds its run, as after job.run(). + for label in ("rowwise", "grouped", "summed"): + assert job.nodes[label].head().status is ViewStatus.ROWS, label + expected = _build(ROWS, InMemoryArrowDatabase()) + expected.run() + for label in ("rowwise", "summed"): + assert ( + job.nodes[label] + .as_table() + .sort_by("g") + .equals(expected.nodes[label].as_table().sort_by("g")) + ), label + + def test_the_cascade_reports_its_counts_on_the_node(self) -> None: + db = InMemoryArrowDatabase() + _build(ROWS[:3], db).run() + CALLS.clear() + job = _build(ROWS, db) + _ = job.nodes["summed"].run() + summary = job.nodes["summed"].last_run + assert summary is not None + assert not summary.load_only + rowwise = summary.nodes["rowwise"] + assert (rowwise.computed, rowwise.reused) == (3, 3) + assert summary.nodes["summed"].computed == 2 + assert "summed" in repr(summary) + + def test_a_diamond_runs_each_ancestor_once(self) -> None: + job = PipelineJob(name="d", database=InMemoryArrowDatabase()) + with job: + top = inc.pod(_source(ROWS), label="top") + left = dbl.pod(top, label="left") + right = neg.pod(top, label="right") + Join()(left, right, label="joined") + view = job.nodes["joined"].run() + assert view.as_table().num_rows == 6 + assert sorted(CALLS) == sorted(["inc"] * 6 + ["dbl"] * 6 + ["neg"] * 6) + + def test_failed_rows_warn_as_a_job_run_does(self) -> None: + job = PipelineJob(name="f", database=InMemoryArrowDatabase()) + with job: + fails_on_20.pod(_source(ROWS), label="node") + with pytest.warns( + RowFailuresWarning, match="1 of 6 rows failed in node 'node'" + ): + view = job.nodes["node"].run() + assert view.as_table().num_rows == 5 + + +class TestState: + """A node that holds a current run is not run again; a stale one is.""" + + def test_a_second_run_of_an_unchanged_node_does_nothing(self) -> None: + db = InMemoryArrowDatabase() + job = _build(ROWS, db) + first = job.nodes["summed"].run() + CALLS.clear() + with ( + mock.patch.object( + InMemoryArrowDatabase, + "get_all_records", + side_effect=AssertionError("store read"), + ), + mock.patch.object( + InMemoryArrowDatabase, + "add_records", + side_effect=AssertionError("store write"), + ), + ): + second = job.nodes["summed"].run() + assert CALLS == [] + assert second.as_table().equals(first.as_table()) + + def test_a_held_ancestor_is_not_run_again(self) -> None: + job = _build(ROWS, InMemoryArrowDatabase()) + _ = job.nodes["rowwise"].run() + stamped = job.nodes["rowwise"].last_modified + CALLS.clear() + _ = job.nodes["summed"].run() + assert job.nodes["rowwise"].last_modified == stamped + assert sorted(CALLS) == ["total", "total"] + + def test_a_node_runs_again_after_its_source_changed(self) -> None: + job = _build(ROWS[:3], InMemoryArrowDatabase()) + _ = job.nodes["summed"].run() + CALLS.clear() + job.bind(sources={next(iter(job.sources)): _source(ROWS)}) + assert job.nodes["summed"].is_stale + view = job.nodes["summed"].run() + assert CALLS.count("inc") == 3 + assert _totals(view.as_table()) == [(0, 63), (1, 93)] + + +class TestLoadOnly: + """``node.run(load_only=True)`` loads through the cascade; its rows have a status.""" + + def test_a_load_only_cascade_computes_and_writes_nothing(self) -> None: + db = InMemoryArrowDatabase() + _build(ROWS[:3], db).run() + CALLS.clear() + before = _snapshot(db) + job = _build(ROWS, db) + view = job.nodes["summed"].run(load_only=True) + assert CALLS == [] + assert _snapshot(db) == before + assert _totals(view.as_table()) == [(0, 22), (1, 11)] + summary = job.nodes["summed"].last_run + assert summary is not None and summary.load_only + assert summary.nodes["rowwise"].not_stored == 3 + rowwise_head = job.nodes["rowwise"].head() + assert rowwise_head.status is ViewStatus.LOADED_ONLY + assert "3 loaded, 3 inputs without a stored result" in rowwise_head.header() + summed_head = job.nodes["summed"].head() + assert summed_head.status is ViewStatus.LOADED_ONLY + assert "inputs incomplete: rowwise" in summed_head.header() + + def test_a_full_run_after_a_load_only_run_computes_what_is_missing(self) -> None: + db = InMemoryArrowDatabase() + _build(ROWS[:3], db).run() + CALLS.clear() + job = _build(ROWS, db) + _ = job.nodes["summed"].run(load_only=True) + assert CALLS == [] + view = job.nodes["summed"].run() + assert CALLS.count("inc") == 3 + assert _totals(view.as_table()) == [(0, 63), (1, 93)] + assert job.nodes["rowwise"].head().status is ViewStatus.ROWS + + def test_a_load_only_run_of_a_load_only_node_returns_what_it_holds(self) -> None: + db = InMemoryArrowDatabase() + _build(ROWS[:3], db).run() + job = _build(ROWS, db) + first = job.nodes["summed"].run(load_only=True) + stamped = job.nodes["rowwise"].last_modified + second = job.nodes["summed"].run(load_only=True) + assert job.nodes["rowwise"].last_modified == stamped + assert second.as_table().equals(first.as_table()) + + def test_a_load_only_cascade_uses_a_fully_run_ancestor(self) -> None: + job = _build(ROWS, InMemoryArrowDatabase()) + _ = job.nodes["rowwise"].run() + stamped = job.nodes["rowwise"].last_modified + _ = job.nodes["summed"].run(load_only=True) + assert job.nodes["rowwise"].last_modified == stamped + + def test_a_load_only_run_that_found_everything_stays_loaded_only(self) -> None: + db = InMemoryArrowDatabase() + _build(ROWS, db).run() + job = _build(ROWS, db) + _ = job.nodes["rowwise"].run(load_only=True) + head = job.nodes["rowwise"].head() + assert head.status is ViewStatus.LOADED_ONLY + assert "6 loaded, 0 inputs without a stored result" in head.header() + + +class TestSideEffects: + def test_a_full_cascade_delivers_a_load_only_cascade_does_not(self) -> None: + db = InMemoryArrowDatabase() + job = _build(ROWS, db, with_tap=True) + _ = job.nodes["summed"].run(load_only=True) + assert DELIVERED == [] + _ = job.nodes["summed"].run() + assert sorted(DELIVERED) == sorted(r["x"] + 1 for r in ROWS) + + +def test_the_cascade_commits_every_write(tmp_path: Path) -> None: + """A store opened again through a new object holds every result and record.""" + path = tmp_path / "store" + job = _build(ROWS, DeltaTableDatabase(path), cache_mode=CacheMode.LOG) + _ = job.nodes["summed"].run() + CALLS.clear() + probe = _build(ROWS, DeltaTableDatabase(path), cache_mode=CacheMode.LOG) + for label, rows in (("rowwise", 6), ("grouped", 2), ("summed", 2)): + records = probe.nodes[label].get_all_records() + assert records is not None and records.num_rows == rows, label + again = _build(ROWS, DeltaTableDatabase(path)) + summary = again.run() + assert CALLS == [] + assert summary.nodes["rowwise"].reused == 6 + assert summary.nodes["summed"].reused == 2 From db1b80af02a809fef323d700f04d0fb8de732a44 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 22:01:20 -0800 Subject: [PATCH 17/48] feat(nodes): node.run() runs the job nodes it depends on, by their state (ITL-1095) A job node's run(*, load_only=False) is state-driven: a node that holds a completed run and is not stale (and, for a full run, was not filled load-only, nor anything above it) returns its rows and runs nothing. Otherwise it runs, after every job node it depends on that holds no current run, each once, on the sync orchestrator restricted to that sub-graph (held rows feed the nodes after them), and commits the run's writes as job.run() does. node.last_run reports the counts; failed rows warn. A load-only run's rows have their own status, ViewStatus.LOADED_ONLY, captioned 'N loaded, M inputs without a stored result' (and the incomplete inputs above). CACHE_ONLY nodes and nodes that cannot compute raise from run() unless load_only. Co-Authored-By: Claude Opus 5.5 --- docs/concepts/function-pods.md | 3 +- src/orcapod/core/nodes/function_node.py | 175 ++++++++++++++++++---- src/orcapod/core/nodes/operator_node.py | 73 +++++++-- src/orcapod/core/nodes/source_node.py | 14 ++ src/orcapod/core/nodes/staleness.py | 43 ++++++ src/orcapod/core/streams/view.py | 14 +- src/orcapod/pipeline/job.py | 31 +--- src/orcapod/pipeline/node_run.py | 145 ++++++++++++++++++ src/orcapod/pipeline/run_summary.py | 71 +++++++-- src/orcapod/pipeline/sync_orchestrator.py | 12 +- tests/test_core/streams/test_run.py | 17 ++- 11 files changed, 514 insertions(+), 84 deletions(-) create mode 100644 src/orcapod/pipeline/node_run.py diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index c01237a4f..cbb4a4415 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -345,7 +345,8 @@ result is read from the store and nothing is computed. A run that raised defines and so does `clear_cache()`; a node whose input changed since its run (an upstream node ran again, or a source was bound again) is stale and empty until it runs again. The stored view, `node.as_source()`, reads every record stored under the node's key, whether or not the node -has run. +has run: the node's stored history, which keeps an entry's earlier result next to its +recomputed one, where a run loads only the latest. `InMemoryArrowDatabase` lasts only as long as the Python process. Use `DeltaTableDatabase("some/directory")` from `orcapod.databases` to keep results on disk diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index ff83b57bf..61250eb25 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -18,7 +18,9 @@ import asyncio import contextlib +import dataclasses import logging +import sys import uuid from abc import abstractmethod from collections import Counter @@ -1574,6 +1576,12 @@ def _set_run_rows( self._cached_output_table = None self._cached_content_hash_column = None + def _holds_run(self, *, load_only: bool) -> bool: + """Whether the node holds a completed run; a load-only one only for ``load_only``.""" + return self._run_entry_counts is not None and ( + load_only or not self._run_load_only + ) + @property def last_run_not_stored(self) -> int: """Inputs the node's last completed run skipped because nothing was stored. @@ -3340,7 +3348,15 @@ def head( return self._head_without_rows( config, ViewStatus.NOT_COMPUTED, note=STALE_NOTE ) - return self._head_of_pairs(list(self._held_pairs()), n, config, origin="held") + head = self._head_of_pairs(list(self._held_pairs()), n, config, origin="held") + if not self._run_load_only: + return head + # A load-only run's rows have their own status: they may lack the inputs it + # skipped (ITL-1095). + note = self._load_only_note() + if head.note: + note += f"; {head.note}" + return dataclasses.replace(head, status=ViewStatus.LOADED_ONLY, note=note) # ------------------------------------------------------------------ # Iteration @@ -3393,44 +3409,149 @@ def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: self.clear_cache() yield from self._held_pairs() - def run(self) -> StreamView: - """Eagerly compute all input data, filling pipeline and result databases, and - return a ``StreamView`` of the node's rows. + def run(self, *, load_only: bool = False) -> StreamView: + """Run the node as a job run would, and return a ``StreamView`` of its rows. - Computes as a job run computes this node, unlike ``as_table()``, which only - reads. A node whose upstream is unavailable (``CACHE_ONLY``) or whose function - pod is a placeholder (``READ_ONLY``) computes nothing and holds no rows; its - stored rows are read with ``as_source()``. + State-driven (developer, November 13, 2026; ITL-1095): a node that holds the rows + of a completed run and is not stale is not run again; the view shows what it + holds, and nothing is computed, read or written. Otherwise the node runs, after + every job node it depends on that holds no current run (each once, in + dependency order, as ``job.run()`` would run them; ``last_run`` reports the + counts), and the run's writes are committed. A node built over streams outside a + job reads its input as it is. + + Args: + load_only: Fill the node, and the nodes it runs first, with what is already + stored and compute nothing (as ``job.run(load_only=True)``). A node + filled this way shows status ``loaded_only`` and is not current for a + later full ``run()``, which computes what it skipped. Returns: - A ``StreamView`` of every row ``as_table()`` then returns. It says "rows", - as ``head()`` then does: the node holds them in memory, whether it computed - them or read them from its store. + A ``StreamView`` of every row ``as_table()`` then returns: status ``rows``, + or ``loaded_only`` after a load-only run. Raises: - RuntimeError: If the node was loaded without a function pod, or - ``load_status`` is UNAVAILABLE (no pod, no DB). - """ + RuntimeError: If no store is attached (``UNAVAILABLE``); if the node cannot + compute (loaded without a function pod, or a placeholder pod) and + ``load_only`` is false; or if its input is unavailable + (``CACHE_ONLY``): read its stored rows with ``as_source()``. + """ + from orcapod.pipeline.node_run import ( + depends_on_job_nodes, + run_with_dependencies, + ) + from orcapod.pipeline.run_summary import warn_row_failures from orcapod.pipeline.serialization import LoadStatus - if self._function_pod is None: - raise self._cannot_compute_error() - self._begin_run() - if self._load_status == LoadStatus.UNAVAILABLE: + status = self._load_status + if status == LoadStatus.UNAVAILABLE: raise self._no_store_error() - if self._load_status in (LoadStatus.CACHE_ONLY, LoadStatus.READ_ONLY): - # CACHE_ONLY: upstream unavailable; computation requires a live input stream. - # READ_ONLY: function pod is a proxy placeholder — cannot compute. - # Neither runs, so the view holds no rows (ITL-1095). - return self._view_of_rows(self.as_table(), "held") + if status == LoadStatus.CACHE_ONLY: + raise RuntimeError( + f"FunctionJobNode {self.label!r}: its input is unavailable, so it has " + "no inputs to run or look up; read its stored results with as_source()." + ) + if not load_only and ( + self._function_pod is None or status == LoadStatus.READ_ONLY + ): + raise self._cannot_compute_error() + if not self.holds_current_run(load_only=load_only): + if depends_on_job_nodes(self): + summary = run_with_dependencies(self, load_only=load_only) + else: + summary = self._run_alone(load_only=load_only) + self._last_run = summary + if not summary.ok: + warn_row_failures(summary, sys._getframe(1)) # pyright: ignore[reportPrivateUsage] # the caller's line is where the warning is shown + return self._run_view() + + def _run_alone(self, *, load_only: bool) -> Any: + """Run this node over its input stream, which is not a job node; commit. + + Returns: + The run's ``RunSummary``. + """ + from dataclasses import replace + + from orcapod.pipeline.run_summary import RunCountingObserver, RunSummary + from orcapod.utils.store_errors import flush_best_effort + if self.is_stale: # Discard any stale in-memory entries before a fresh computation run # so that rerunning does not mix old cached entries with new results. self.clear_cache() - self.execute(self._input_stream) - # The rows are the ones held in memory, as head() names them afterwards: some - # may have been read from the result store rather than computed. - return self._view_of_rows(self.as_table(), "held") + counter = RunCountingObserver() + stores = [ + store + for store in ( + self._pipeline_database, + getattr(self._cached_function_pod, "result_database", None), + ) + if store is not None + ] + try: + _ = self.execute(self._input_stream, observer=counter, load_only=load_only) + except Exception as run_error: + flush_best_effort(run_error, *stores) + raise + for store in stores: + store.flush() + node_summary = replace( + counter.node_summary(self.label), not_stored=self.last_run_not_stored + ) + base_path = getattr(self._pipeline_database, "base_path", ()) + return RunSummary( + "/".join(base_path), [node_summary], load_only=load_only, target=self.label + ) + + def _incomplete_inputs(self) -> list[str]: + """Labels of the function nodes above this one whose load-only run skipped inputs.""" + labels: list[str] = [] + seen: list[Any] = [] + pending: list[Any] = list(self.upstreams) + while pending: + node = pending.pop() + if any(node is other for other in seen): + continue + seen.append(node) + if isinstance(node, FunctionJobNode) and node._run_load_only: + if node._run_not_stored: + labels.append(node.label) + pending.extend(getattr(node, "upstreams", ())) + return sorted(labels) + + def _load_only_note(self) -> str: + """The caption of a node filled by a load-only run (ITL-1095).""" + note = ( + f"loaded only: {self._run_loaded_count():,} loaded, " + f"{self._run_not_stored:,} inputs without a stored result" + ) + incomplete = self._incomplete_inputs() + if incomplete: + note += f"; inputs incomplete: {', '.join(incomplete)}" + return note + + def _run_loaded_count(self) -> int: + """How many inputs of the last run had a stored result (loaded).""" + counts = self._run_entry_counts + return 0 if counts is None else sum(counts.values()) + + def _run_view(self) -> StreamView: + """The ``run()`` view of the node's rows, with the status of its last run.""" + table = self.as_table() + if not self._run_load_only: + # The rows are the ones held in memory, as head() names them afterwards: + # some may have been read from the result store rather than computed. + return self._view_of_rows(table, "held") + return StreamView.of_rows( + self, + table, + tag_columns=self.keys()[0], + total_rows=table.num_rows, + status=ViewStatus.LOADED_ONLY, + note=self._load_only_note(), + display_all_rows=False, + ) # ------------------------------------------------------------------ # Async channel execution diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 16c3321d6..a9aa8ec30 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -17,6 +17,7 @@ from __future__ import annotations import asyncio +import sys import logging from abc import abstractmethod from collections.abc import Collection, Iterator, Mapping, Sequence @@ -761,6 +762,7 @@ def clear_cache(self) -> None: super().clear_cache() # clears _node_identity_path_cache + _update_modified_time self._cached_output_stream = None self._cached_output_table = None + self._run_load_only = False # No content established: the node, and every node downstream, is stale. self._set_modified_time(None) @@ -1350,6 +1352,7 @@ def execute( ctx_obs.on_node_start(node_label, node_hash) + self._run_load_only = load_only # Check REPLAY cache first cached_output = self.get_cached_output() if cached_output is not None: @@ -1385,13 +1388,17 @@ def execute( ctx_obs.on_node_end(node_label, node_hash) return output - def _compute_and_store(self) -> None: - """Compute operator output, optionally store in DB.""" + def _compute_and_store(self, *, store: bool = True) -> None: + """Compute operator output, optionally store in DB. + + Args: + store: Whether a ``LOG`` node stores the output (not in a load-only run). + """ self._cached_output_stream = self._operator.process( *self._input_streams, ) - if self._cache_mode == CacheMode.OFF: + if self._cache_mode == CacheMode.OFF or not store: self._update_modified_time() return @@ -1432,12 +1439,15 @@ def _replay_from_cache(self) -> None: ) self._update_modified_time() - def run(self) -> StreamView: - """Execute the operator according to the current cache mode, and return a - ``StreamView`` of the node's rows. + def run(self, *, load_only: bool = False) -> StreamView: + """Run the node as a job run would, and return a ``StreamView`` of its rows. - Computes as a job run computes this node, unlike ``as_table()``, which only - reads. + State-driven (developer, November 13, 2026; ITL-1095): a node that holds the + output of a completed run and is not stale is not run again; the view shows it. + Otherwise the node runs, after every job node it depends on that holds no + current run (each once, in dependency order, as ``job.run()`` would run them; + ``last_run`` reports the counts), and the run's writes are committed. A node + built over streams outside a job runs its operator on them directly. Without a database: Always compute via the operator's ``process()`` method. @@ -1447,10 +1457,35 @@ def run(self) -> StreamView: - **LOG**: always compute, write results to DB. - **REPLAY**: skip computation, load from DB. + Args: + load_only: As ``job.run(load_only=True)``: the operator runs on the rows + that reach it, a ``LOG`` node writes nothing, and the nodes run first + load their stored results only. + Returns: A ``StreamView`` of every row ``as_table()`` then returns, naming its rows as ``head()`` then names them. + + Raises: + RuntimeError: If the node was loaded by ``PipelineJob.load`` and + ``load_only`` is false. """ + from orcapod.pipeline.node_run import ( + depends_on_job_nodes, + run_with_dependencies, + ) + from orcapod.pipeline.run_summary import warn_row_failures + + if self._loaded and not load_only: + raise self._cannot_compute_error() + if self.holds_current_run(load_only=load_only): + return self._run_view(computed=False) + if depends_on_job_nodes(self): + summary = run_with_dependencies(self, load_only=load_only) + self._last_run = summary + if not summary.ok: + warn_row_failures(summary, sys._getframe(1)) # pyright: ignore[reportPrivateUsage] # the caller's line is where the warning is shown + return self._run_view(computed=False) if self._loaded: raise self._cannot_compute_error() if self.is_stale: @@ -1462,14 +1497,23 @@ def run(self) -> StreamView: if self._cache_mode == CacheMode.REPLAY: self._replay_from_cache() else: - self._compute_and_store() + self._compute_and_store(store=not load_only) computed = True + self._run_load_only = load_only else: self._cached_output_stream = self._operator.process( *self._input_streams, ) self._update_modified_time() computed = True + return self._run_view(computed=computed) + + def _run_view(self, *, computed: bool) -> StreamView: + """The ``run()`` view of the node's output. + + Args: + computed: Whether this ``run()`` computed the output lazily just now. + """ # Read the kept output itself: a lazy input that finishes computing during this # read stamps itself after the node, which would make as_table() judge it stale. assert self._cached_output_stream is not None @@ -1486,6 +1530,16 @@ def run(self) -> StreamView: origin = "computed" if kept is None else head_of(kept.head(1), self).origin return self._view_of_rows(table, origin) + # Whether the output the node holds came from a load-only run (ITL-1095): its + # input may lack rows, so a full run does not take it as complete. + _run_load_only: bool = False + + def _holds_run(self, *, load_only: bool) -> bool: + """Whether the node holds a run's output; a load-only run's only for ``load_only``.""" + return self._cached_output_stream is not None and ( + load_only or not self._run_load_only + ) + def _current_output(self) -> StreamProtocol | None: """The output of the node's current run, or ``None`` when it holds none. @@ -1710,6 +1764,7 @@ async def async_execute( try: ctx_obs.on_node_start(node_label, node_hash) + self._run_load_only = load_only if ( self._pipeline_database is not None diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index a7535d2b1..07537c705 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -591,6 +591,20 @@ def __init__( # bound_source property — explicit binding with cache invalidation # ------------------------------------------------------------------ + def run(self, *, load_only: bool = False) -> StreamView: + """Read the bound source's rows and return a ``StreamView`` of all of them. + + A source is read, never computed, so ``load_only`` changes nothing; a job + node's ``run()`` takes it (ITL-1095). + + Args: + load_only: Accepted for every job node; a source reads its rows either way. + + Raises: + UnboundSourceError: When no concrete source is bound. + """ + return super().run() + @property def bound_source(self) -> StreamProtocol | None: """The concrete stream currently bound to this slot, or ``None``.""" diff --git a/src/orcapod/core/nodes/staleness.py b/src/orcapod/core/nodes/staleness.py index 06efbe5b8..841a96fd0 100644 --- a/src/orcapod/core/nodes/staleness.py +++ b/src/orcapod/core/nodes/staleness.py @@ -20,6 +20,20 @@ class RunHoldingNode: their own ``is_stale`` is redefined by ITL-1091. """ + # The summary of the node's last ``run()`` (with the nodes it ran first); a class + # default so nodes rebuilt from a saved descriptor start without one. + _last_run: Any = None + + @property + def last_run(self) -> Any: + """The ``RunSummary`` of this node's last ``run()``, or ``None``. + + ``node.run()`` runs the node and the job nodes it depends on (ITL-1095); this is + what that run did, per function node, as ``job.last_run`` is for ``job.run()``. + A ``run()`` that found the node current ran nothing and leaves it unchanged. + """ + return self._last_run + @property def is_stale(self) -> bool: """Whether the rows this node holds are out of date, or it holds none. @@ -32,6 +46,35 @@ def is_stale(self) -> bool: own: bool = super().is_stale # pyright: ignore[reportAttributeAccessIssue] # the stream base after this mixin in the MRO defines it return own or upstream_job_node_stale(self.upstreams) # pyright: ignore[reportAttributeAccessIssue] # every node class this is mixed into has upstreams + def holds_current_run(self, *, load_only: bool = False) -> bool: + """Whether the node holds the rows of a run that ``run()`` would not redo. + + A node that is not stale and holds the rows of a completed run in this process + is current: ``run()`` returns them without running it (developer, November 13, + 2026, ITL-1095). A node filled by a load-only run is current only for another + load-only run: a full run computes the inputs it skipped. So is every node + whose rows came from such a node: a node is current only if every job node + above it is current for the same kind of run. + + Args: + load_only: Whether the question is for a load-only run. + + Returns: + ``True`` if the node holds a current run for such a run. + """ + if not self._holds_run(load_only=load_only) or self.is_stale: + return False + upstreams: tuple[Any, ...] = getattr(self, "upstreams", ()) + return all( + upstream.holds_current_run(load_only=load_only) + for upstream in upstreams + if isinstance(upstream, RunHoldingNode) + ) + + def _holds_run(self, *, load_only: bool) -> bool: + """Whether the node holds the rows of a completed run good for such a run.""" + raise NotImplementedError + def upstream_job_node_stale(upstreams: Iterable[Any]) -> bool: """Return whether any job node in ``upstreams`` that holds run rows is stale. diff --git a/src/orcapod/core/streams/view.py b/src/orcapod/core/streams/view.py index f440bde36..d81e93c16 100644 --- a/src/orcapod/core/streams/view.py +++ b/src/orcapod/core/streams/view.py @@ -45,6 +45,9 @@ class ViewStatus(str, Enum): ROWS: The rows are the first rows of the stream's complete content. Zero rows means the content is empty. PARTIAL: A computation stopped part way; the rows are those computed so far. + LOADED_ONLY: A job node filled by a load-only run (``run(load_only=True)``): + the rows are the stored results of its inputs; inputs without one were + skipped, so the rows may be fewer than a full run gives (ITL-1095). NOT_COMPUTED: Nothing is computed, or what was computed is out of date. NOTHING_STORED: A stored view (``node.as_source()``) whose store holds no rows for its node yet. @@ -54,6 +57,7 @@ class ViewStatus(str, Enum): ROWS = "rows" PARTIAL = "partial" + LOADED_ONLY = "loaded_only" NOT_COMPUTED = "not_computed" NOTHING_STORED = "nothing_stored" UNAVAILABLE = "unavailable" @@ -372,7 +376,7 @@ def _is_window(self) -> bool: can be named by position: a ``ROWS`` view with no ``description``, unsorted or sorted among all of the stream's rows.""" return ( - self.status is ViewStatus.ROWS + self.status in (ViewStatus.ROWS, ViewStatus.LOADED_ONLY) and self.description is None and (not self.sorted_by_tags or self._total_is_exact()) ) @@ -392,7 +396,11 @@ def _holds_every_row(self) -> bool: def _cuttable(self) -> StreamView: """Return this view in a form whose rows can be cut by position: itself when its rows are a run of the stream's rows (or of a pool), else a pool of its rows.""" - if self.status not in (ViewStatus.ROWS, ViewStatus.PARTIAL): + if self.status not in ( + ViewStatus.ROWS, + ViewStatus.PARTIAL, + ViewStatus.LOADED_ONLY, + ): return self if self.pool is not None or self._is_window(): return self @@ -423,7 +431,7 @@ def _describe(self) -> str: shown = self.table.num_rows if self.pool is not None: return _window_text(self.pool, shown, self.start, self.pool_rows) - if self.status is ViewStatus.ROWS: + if self.status in (ViewStatus.ROWS, ViewStatus.LOADED_ONLY): text = self._describe_rows(shown) elif self.status is ViewStatus.PARTIAL: total = self.total_rows if self.total_rows is not None else shown diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index eee77a79a..30cd42bbe 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -5,7 +5,6 @@ import contextlib import logging import sys -import warnings from collections.abc import Iterator from dataclasses import dataclass from typing import TYPE_CHECKING, Any @@ -22,7 +21,6 @@ unclaimed_tables_written_before_v03, ) from orcapod.errors import ( - RowFailuresWarning, StoreError, StoreReadError, StoreWriteError, @@ -34,6 +32,7 @@ RunCountingObserver, RunSummary, run_node_summaries, + warn_row_failures, ) from orcapod.protocols import core_protocols as cp from orcapod.databases.utils import as_table_committing @@ -42,7 +41,6 @@ if TYPE_CHECKING: from pathlib import Path - from types import FrameType from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator from orcapod.pipeline.execution_context import ExecutionContext @@ -98,29 +96,6 @@ def _with_counter( return CompositeObserver(observer, counter) # pyright: ignore[reportReturnType] # forwards every hook; its **kwargs signatures do not match the protocol structurally -def _warn_row_failures(summary: RunSummary, caller: FrameType) -> None: - """Issue the run's ``RowFailuresWarning``, shown after every run with failed rows. - - Python's default filter shows a warning once per text and line, so a second identical - failing run from the same line (re-running a notebook cell) would show nothing. A - fresh registry for each run keeps every run's warning, while the user's filters - (``ignore``, ``error``) still apply. - - Args: - summary: The run's summary, with failed rows. - caller: The frame that called ``run``; the warning is shown at its line. - """ - warnings.warn_explicit( - summary.failures_message(), - RowFailuresWarning, - caller.f_code.co_filename, - caller.f_lineno, - module=caller.f_globals.get("__name__"), - registry=None, - module_globals=caller.f_globals, - ) - - class PipelineJob(AbstractPipelineBase[JobNode]): """Pipeline + source bindings + execution context. @@ -967,7 +942,7 @@ def run( ) summary = self._finish_run(prepared, counter, load_only=load_only) if not summary.ok: - _warn_row_failures(summary, sys._getframe(1)) # pyright: ignore[reportPrivateUsage] # the caller's line is where the warning is shown + warn_row_failures(summary, sys._getframe(1)) # pyright: ignore[reportPrivateUsage] # the caller's line is where the warning is shown return summary async def run_async( @@ -1029,7 +1004,7 @@ async def run_async( ) summary = self._finish_run(prepared, counter, load_only=load_only) if not summary.ok: - _warn_row_failures(summary, sys._getframe(1)) # pyright: ignore[reportPrivateUsage] # the caller's line is where the warning is shown + warn_row_failures(summary, sys._getframe(1)) # pyright: ignore[reportPrivateUsage] # the caller's line is where the warning is shown return summary def _prepare_run(self, commit_every_rows: int) -> "_PreparedRun": diff --git a/src/orcapod/pipeline/node_run.py b/src/orcapod/pipeline/node_run.py new file mode 100644 index 000000000..3cdf8d468 --- /dev/null +++ b/src/orcapod/pipeline/node_run.py @@ -0,0 +1,145 @@ +"""``node.run()`` of a job node: run it and the job nodes it depends on (ITL-1095). + +Developer, November 13, 2026: "given that the initially loaded job is empty, I think it's +fair that node.run will cascade node.run on all nodes it depends on recursively. +Critically, node.run(load_only=True) would trigger the upstream node.run also with +load_only=True." And: "if it's not stale, it shouldn't even trigger a run and simply +return what's in the stream cached already." + +The cascade is ``job.run()`` restricted to the node's dependencies, on the synchronous +orchestrator: a dependency that holds a current run is not run again (its rows feed the +nodes after it), every other one runs once, in dependency order, and the run ends with +the commit ``job.run()`` makes. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from orcapod.errors import StoreError, StoreWriteError +from orcapod.pipeline.run_summary import ( + RunCountingObserver, + RunSummary, + run_node_summaries, +) +from orcapod.utils.store_errors import flush_best_effort + +if TYPE_CHECKING: + from orcapod.protocols.database_protocols import ArrowDatabaseProtocol + + +def _is_job_node(stream: Any) -> bool: + """Whether ``stream`` is a node of a job (as opposed to a stream outside any job).""" + from orcapod.core.nodes.source_node import SourceJobNode + from orcapod.core.nodes.staleness import RunHoldingNode + from orcapod.side_effects import SideEffectJobNode + + return isinstance(stream, (RunHoldingNode, SideEffectJobNode, SourceJobNode)) + + +def depends_on_job_nodes(node: Any) -> bool: + """Whether any direct input of ``node`` is a job node. + + A node built directly over streams outside a job (a source, a pod's stream) has + nothing to run first: its ``run()`` reads its input as it always has. + """ + return any(_is_job_node(upstream) for upstream in node.upstreams) + + +def _stores(nodes: list[Any]) -> list[ArrowDatabaseProtocol]: + """The distinct stores the given nodes write to: pipeline and result databases.""" + found: list[ArrowDatabaseProtocol] = [] + for node in nodes: + pipeline_database = getattr(node, "_pipeline_database", None) + cached_pod = getattr(node, "_cached_function_pod", None) + result_database = getattr(cached_pod, "result_database", None) + for database in (pipeline_database, result_database): + if database is not None and not any(database is d for d in found): + found.append(database) + return found + + +def _job_name(node: Any) -> str: + """The name of the job ``node`` belongs to: its pipeline database's location.""" + database = getattr(node, "_pipeline_database", None) + base_path: tuple[str, ...] = getattr(database, "base_path", ()) + return "/".join(base_path) + + +def run_with_dependencies(target: Any, *, load_only: bool) -> RunSummary: + """Run ``target`` and every job node it depends on that holds no current run. + + The dependencies are found through ``upstreams``. A dependency that holds a current + run (``holds_current_run``) is not run: its rows feed the nodes after it. A stream + outside the job is read. Every other node runs once, in dependency order, on the + ``SyncPipelineOrchestrator`` (error policy ``"continue"``), as ``job.run()`` would + run that sub-graph; with ``load_only`` as ``job.run(load_only=True)`` would. The run + ends with a commit of every store it wrote to, also, best-effort, when it fails. + + Args: + target: The job node whose ``run()`` this is. + load_only: Whether to load stored results only. + + Returns: + The run's summary, one entry per function node that ran. + + Raises: + StoreWriteError: If committing the run's writes failed. + """ + from orcapod.core.nodes.source_node import SourceJobNode + from orcapod.core.nodes.staleness import RunHoldingNode + from orcapod.pipeline.dag import OrcaDAG + from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator + + dag: OrcaDAG[Any] = OrcaDAG() + held: dict[Any, list[Any]] = {} + seen: list[Any] = [] + + def visit(node: Any) -> None: + """Add ``node`` and, unless it is held or a source, what it depends on.""" + if any(node is other for other in seen): + return + seen.append(node) + dag.add_node(node) + if ( + node is not target + and isinstance(node, RunHoldingNode) + and node.holds_current_run(load_only=load_only) + ): + held[node] = list(node.iter_data()) + return + if isinstance(node, SourceJobNode): + return + if not _is_job_node(node): + # A stream outside the job: its rows, read as the node's run reads them. + held[node] = list(node.iter_data()) + return + for upstream in node.upstreams: + visit(upstream) + dag.add_edge(upstream, node) + + visit(target) + run_nodes = [node for node in seen if not any(node is h for h in held)] + stores = _stores(run_nodes) + counter = RunCountingObserver() + try: + _ = SyncPipelineOrchestrator().run( + dag, observer=counter, load_only=load_only, held_rows=held + ) + except Exception as run_error: + # Commit the writes that succeeded before the failure, as job.run() does. + flush_best_effort(run_error, *stores) + raise + for store in stores: + try: + store.flush() + except StoreError: + raise + except Exception as exc: + raise StoreWriteError(store.base_path, flush=True) from exc + return RunSummary( + _job_name(target), + run_node_summaries(dag, counter, skip=list(held)), + load_only=load_only, + target=target.label, + ) diff --git a/src/orcapod/pipeline/run_summary.py b/src/orcapod/pipeline/run_summary.py index 84124357d..ee48b49f3 100644 --- a/src/orcapod/pipeline/run_summary.py +++ b/src/orcapod/pipeline/run_summary.py @@ -18,7 +18,8 @@ from orcapod.pipeline.observer import NoOpLogger if TYPE_CHECKING: - from collections.abc import Mapping, Sequence + from collections.abc import Collection, Mapping, Sequence + from types import FrameType from orcapod.protocols.core_protocols import DataProtocol, TagProtocol from orcapod.types import SchemaLike @@ -70,6 +71,8 @@ class RunSummary: nodes: Per function node, in run order, keyed by label. load_only: Whether the run was a load-only run (``job.run(load_only=True)``): it computed nothing; ``reused`` counts the rows it loaded. + target: For the run of one node and the nodes it depends on + (``node.run()``), that node's label; ``None`` for a job run. """ def __init__( @@ -78,6 +81,7 @@ def __init__( nodes: Sequence[NodeRunSummary], *, load_only: bool = False, + target: str | None = None, ) -> None: """Build the summary. @@ -85,10 +89,12 @@ def __init__( job_name: The job's name. nodes: One entry per function node of the run, in run order. load_only: Whether the run loaded stored results only. + target: The label of the node whose ``run()`` this was, if any. """ self.job_name: str = job_name self.nodes: Mapping[str, NodeRunSummary] = {node.label: node for node in nodes} self.load_only: bool = load_only + self.target: str | None = target @property def failed_rows(self) -> int: @@ -109,7 +115,7 @@ def failures_message(self) -> str: return " ".join( f"{node.failed} of {_rows(node.rows)} failed in node {node.label!r} " f"(first error: {_error_line(node.first_error)}). " - f"Full traceback: {_traceback_call(node.label)}." + f"Full traceback: {_traceback_call(node.label, self.target)}." for node in self.nodes.values() if node.failed ) @@ -137,7 +143,12 @@ def print_traceback(self, label: str | None = None) -> None: print(f"No rows failed{where} in this run.") def _headline(self) -> str: - """The first line: the job and whether any row failed.""" + """The first line: the job (or node) and whether any row failed.""" + subject = ( + f"job {self.job_name!r}" + if self.target is None + else f"node {self.target!r} and the nodes it depends on" + ) if self.load_only: skipped = sum(node.not_stored for node in self.nodes.values()) outcome = ( @@ -145,12 +156,12 @@ def _headline(self) -> str: if skipped else "every input had a stored result." ) - return f"Load-only run of job {self.job_name!r}: {outcome}" + return f"Load-only run of {subject}: {outcome}" if self.failed_rows: outcome = f"{_rows(self.failed_rows)} failed." else: outcome = "all rows succeeded." - return f"Run of job {self.job_name!r}: {outcome}" + return f"Run of {subject}: {outcome}" def __repr__(self) -> str: """The summary as text: one line per node, and the first error of a failing one.""" @@ -161,7 +172,9 @@ def __repr__(self) -> str: ) if node.failed: lines.append(f" first error: {_error_line(node.first_error)}") - lines.append(f" Full traceback: {_traceback_call(node.label)}") + lines.append( + f" Full traceback: {_traceback_call(node.label, self.target)}" + ) return "\n".join(lines) def _repr_html_(self) -> str: @@ -188,7 +201,7 @@ def _repr_html_(self) -> str: if node.failed: error = ( f"{html.escape(_error_line(node.first_error))}
" - f"{html.escape(_traceback_call(node.label))}" + f"{html.escape(_traceback_call(node.label, self.target))}" ) body.append( "" @@ -255,12 +268,41 @@ def _error_line(error: BaseException | None) -> str: return f"{type(error).__name__}: {first}" if first else type(error).__name__ -def _traceback_call(label: str) -> str: +def _traceback_call(label: str, target: str | None = None) -> str: """The call that prints a node's traceback, with the label as a valid string literal. - A JSON string is a valid Python string literal, and keeps double quotes. + A JSON string is a valid Python string literal, and keeps double quotes. A node + run's summary is kept on its node (``target``), a job run's on the job. + """ + owner = "job" if target is None else f"job.nodes[{json.dumps(target)}]" + return f"{owner}.last_run.print_traceback({json.dumps(label)})" + + +def warn_row_failures(summary: RunSummary, caller: FrameType) -> None: + """Issue the run's ``RowFailuresWarning``, shown after every run with failed rows. + + Python's default filter shows a warning once per text and line, so a second identical + failing run from the same line (re-running a notebook cell) would show nothing. A + fresh registry for each run keeps every run's warning, while the user's filters + (``ignore``, ``error``) still apply. + + Args: + summary: The run's summary, with failed rows. + caller: The frame that called ``run``; the warning is shown at its line. """ - return f"job.last_run.print_traceback({json.dumps(label)})" + import warnings + + from orcapod.errors import RowFailuresWarning + + warnings.warn_explicit( + summary.failures_message(), + RowFailuresWarning, + caller.f_code.co_filename, + caller.f_lineno, + module=caller.f_globals.get("__name__"), + registry=None, + module_globals=caller.f_globals, + ) class RunCountingObserver: @@ -351,7 +393,9 @@ def create_data_logger(self, tag: TagProtocol, data: DataProtocol) -> Any: return _NOOP_LOGGER -def run_node_summaries(dag: Any, counter: RunCountingObserver) -> list[NodeRunSummary]: +def run_node_summaries( + dag: Any, counter: RunCountingObserver, skip: Collection[Any] = () +) -> list[NodeRunSummary]: """One summary per function node of a finished run, in run order. The counts come from ``counter``; a load-only run's skipped inputs come from each @@ -361,6 +405,9 @@ def run_node_summaries(dag: Any, counter: RunCountingObserver) -> list[NodeRunSu Args: dag: The run's node graph (``topological_sort``, ``predecessors``). counter: The observer that counted the run's rows. + skip: Nodes of ``dag`` the run did not run (their rows were held): they get no + summary, but inputs they skipped in an earlier load-only run still make the + nodes below them incomplete. Returns: The summaries of the run's function nodes, in run order. @@ -383,6 +430,8 @@ def run_node_summaries(dag: Any, counter: RunCountingObserver) -> list[NodeRunSu if getattr(upstream, "last_run_not_stored", 0): above.add(id(upstream)) skipping_above[id(node)] = above + if any(node is skipped for skipped in skip): + continue if not (is_function_node(node) or is_side_effect_function_node(node)): continue labels = tuple( diff --git a/src/orcapod/pipeline/sync_orchestrator.py b/src/orcapod/pipeline/sync_orchestrator.py index af57145b3..a6515de5e 100644 --- a/src/orcapod/pipeline/sync_orchestrator.py +++ b/src/orcapod/pipeline/sync_orchestrator.py @@ -23,6 +23,8 @@ ) if TYPE_CHECKING: + from collections.abc import Mapping + from orcapod.pipeline.dag import GraphProtocol from orcapod.protocols.observability_protocols import ExecutionObserverProtocol from orcapod.protocols.core_protocols import DataProtocol, TagProtocol @@ -56,6 +58,7 @@ def run( run_id: str | None = None, pipeline_uri: str = "", load_only: bool = False, + held_rows: Mapping[Any, list[tuple[TagProtocol, DataProtocol]]] | None = None, ) -> OrchestratorResult: """Execute the node graph synchronously. @@ -74,6 +77,10 @@ def run( (ITL-1095): function nodes load their inputs' stored results and skip the others, operators run without storing, and a side-effect node passes its rows on without being called (nothing is delivered). + held_rows: Nodes of ``graph`` that are not run: each one's rows (the rows + of its current run, or of a stream outside the job) feed its + successors as if it had just run. Used by ``node.run()``, which runs + only the nodes it depends on that hold no current run (ITL-1095). Returns: OrchestratorResult with node outputs. @@ -91,9 +98,12 @@ def run( buffers: dict[Any, list[tuple[TagProtocol, DataProtocol]]] = {} processed: set[Any] = set() + held = held_rows if held_rows is not None else {} for node in topo_order: current = node - if is_source_node(node): + if node in held: + buffers[node] = list(held[node]) + elif is_source_node(node): buffers[node] = node.execute(observer=effective_observer) elif is_function_node(node) or is_side_effect_function_node(node): upstream_buf = self._gather_upstream(node, graph, buffers) diff --git a/tests/test_core/streams/test_run.py b/tests/test_core/streams/test_run.py index 54da7aecc..533e4785f 100644 --- a/tests/test_core/streams/test_run.py +++ b/tests/test_core/streams/test_run.py @@ -107,12 +107,21 @@ def _all_stream_classes() -> list[type]: @pytest.mark.parametrize("cls", _all_stream_classes(), ids=lambda c: c.__qualname__) -def test_every_stream_class_has_run_without_arguments(cls: type) -> None: - """``run`` exists on every stream class and takes no argument (developer decision).""" +def test_every_stream_class_has_run_without_positional_arguments(cls: type) -> None: + """``run`` exists on every stream class and takes no positional argument (developer + decision, ITL-1081); a job node's ``run`` takes one keyword-only argument, + ``load_only`` (ITL-1095).""" run = getattr(cls, "run", None) assert callable(run), f"{cls.__qualname__} has no run()" - parameters = list(inspect.signature(run).parameters) - assert parameters == ["self"], f"{cls.__qualname__}.run{parameters}" + parameters = inspect.signature(run).parameters + names = list(parameters) + if cls.__name__ in ("FunctionJobNode", "OperatorJobNode", "SourceJobNode"): + assert names == ["self", "load_only"], f"{cls.__qualname__}.run{names}" + load_only = parameters["load_only"] + assert load_only.kind is inspect.Parameter.KEYWORD_ONLY + assert load_only.default is False + else: + assert names == ["self"], f"{cls.__qualname__}.run{names}" def test_the_stream_protocol_declares_run() -> None: From 48a2015557ee74cf16102c0bb760537af47244b8 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 22:02:30 -0800 Subject: [PATCH 18/48] fix(pipeline): a run whose end-of-run commit fails leaves no node holding its rows (ITL-1095) job.run() (both orchestrators) and the node.run() cascade clear every function and operator job node of the run (not run, stale) before the commit's error propagates. A load-only run writes nothing, so its commit has nothing to fail on. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/pipeline/job.py | 4 ++ src/orcapod/pipeline/node_run.py | 22 ++++++ tests/test_pipeline/test_node_run_cascade.py | 73 ++++++++++++++++++++ 3 files changed, 99 insertions(+) diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 30cd42bbe..e36f6ca08 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -28,6 +28,7 @@ ) from orcapod.pipeline.base import AbstractPipelineBase from orcapod.pipeline.dag import OrcaDAG +from orcapod.pipeline.node_run import clear_run_rows from orcapod.pipeline.run_summary import ( RunCountingObserver, RunSummary, @@ -1237,10 +1238,13 @@ def _finish_run( except StoreReadError as exc: # A stored table could not be read: report the read failure itself. exc.add_note(pending_note) + # A run whose commit failed did not complete: no node holds its rows. + clear_run_rows(prepared.exec_dag.nodes()) raise except Exception as exc: flush_error = StoreWriteError(pipeline_db.base_path, flush=True) flush_error.add_note(pending_note) + clear_run_rows(prepared.exec_dag.nodes()) raise flush_error from exc # Mutate self in place. diff --git a/src/orcapod/pipeline/node_run.py b/src/orcapod/pipeline/node_run.py index 3cdf8d468..bfeae0e43 100644 --- a/src/orcapod/pipeline/node_run.py +++ b/src/orcapod/pipeline/node_run.py @@ -14,6 +14,7 @@ from __future__ import annotations +from collections.abc import Iterable from typing import TYPE_CHECKING, Any from orcapod.errors import StoreError, StoreWriteError @@ -66,6 +67,24 @@ def _job_name(node: Any) -> str: return "/".join(base_path) +def clear_run_rows(nodes: Iterable[Any]) -> None: + """Drop the rows of a run whose end-of-run commit failed from every node it filled. + + A run that raised defines no rows (ITL-1093): each function and operator job node of + the run is cleared (not run, stale) before the commit's error propagates. What was + committed stays stored. A load-only run writes nothing, so its commit has nothing + to fail on. + + Args: + nodes: The nodes of the run. + """ + from orcapod.core.nodes.staleness import RunHoldingNode + + for node in nodes: + if isinstance(node, RunHoldingNode): + node.clear_cache() # pyright: ignore[reportAttributeAccessIssue] # every RunHoldingNode is a job node with clear_cache + + def run_with_dependencies(target: Any, *, load_only: bool) -> RunSummary: """Run ``target`` and every job node it depends on that holds no current run. @@ -134,8 +153,11 @@ def visit(node: Any) -> None: try: store.flush() except StoreError: + # A run whose commit failed did not complete: no node holds its rows. + clear_run_rows(run_nodes) raise except Exception as exc: + clear_run_rows(run_nodes) raise StoreWriteError(store.base_path, flush=True) from exc return RunSummary( _job_name(target), diff --git a/tests/test_pipeline/test_node_run_cascade.py b/tests/test_pipeline/test_node_run_cascade.py index f986068ab..4d518933b 100644 --- a/tests/test_pipeline/test_node_run_cascade.py +++ b/tests/test_pipeline/test_node_run_cascade.py @@ -312,3 +312,76 @@ def test_the_cascade_commits_every_write(tmp_path: Path) -> None: assert CALLS == [] assert summary.nodes["rowwise"].reused == 6 assert summary.nodes["summed"].reused == 2 + + +class TestFailedFinalCommit: + """A run whose end-of-run commit fails leaves no node holding its rows.""" + + @staticmethod + def _flush_calls(run: Any) -> int: + """How many flushes ``run()`` makes on an in-memory store.""" + calls = 0 + original = InMemoryArrowDatabase.flush + + def counting(self: Any) -> None: + nonlocal calls + calls += 1 + original(self) + + with mock.patch.object(InMemoryArrowDatabase, "flush", counting): + run() + return calls + + @staticmethod + def _failing_last_flush(total_calls: int) -> Any: + """A flush that fails on call number ``total_calls`` (the final commit).""" + calls = 0 + original = InMemoryArrowDatabase.flush + + def flush(self: Any) -> None: + nonlocal calls + calls += 1 + if calls == total_calls: + raise OSError("disk full") + original(self) + + return mock.patch.object(InMemoryArrowDatabase, "flush", flush) + + @pytest.mark.parametrize("orchestrator", ["sync", "async"]) + def test_a_job_run_whose_commit_fails_leaves_every_node_empty( + self, orchestrator: str + ) -> None: + from orcapod.errors import StoreWriteError + from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator + + kinds = {"sync": SyncPipelineOrchestrator, "async": AsyncPipelineOrchestrator} + total = self._flush_calls( + lambda: _build(ROWS, InMemoryArrowDatabase()).run( + orchestrator=kinds[orchestrator]() + ) + ) + job = _build(ROWS, InMemoryArrowDatabase()) + with self._failing_last_flush(total), pytest.raises(StoreWriteError): + job.run(orchestrator=kinds[orchestrator]()) + for label in ("rowwise", "grouped", "summed"): + node = job.nodes[label] + assert node.as_table().num_rows == 0, label + assert node.head().status is ViewStatus.NOT_COMPUTED, label + assert node.is_stale, label + job.run(orchestrator=kinds[orchestrator]()) + assert _totals(job.nodes["summed"].as_table()) == [(0, 63), (1, 93)] + + def test_a_cascade_whose_commit_fails_leaves_every_node_empty(self) -> None: + from orcapod.errors import StoreWriteError + + total = self._flush_calls( + lambda: _build(ROWS, InMemoryArrowDatabase()).nodes["summed"].run() + ) + job = _build(ROWS, InMemoryArrowDatabase()) + with self._failing_last_flush(total), pytest.raises(StoreWriteError): + job.nodes["summed"].run() + for label in ("rowwise", "grouped", "summed"): + assert job.nodes[label].as_table().num_rows == 0, label + assert job.nodes[label].is_stale, label + view = job.nodes["summed"].run() + assert _totals(view.as_table()) == [(0, 63), (1, 93)] From 910842353ccc4d32feee1414fe9573609fb4bc47 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 22:03:26 -0800 Subject: [PATCH 19/48] docs: running one node, node states, and the run() argument rule (ITL-1095) Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 13 ++++++++++--- CLAUDE.md | 13 ++++++++++--- changelog.d/ITL-1095.md | 4 ++++ design_issues/ITL-1095-CR1.md | 8 ++++++-- docs/concepts/function-pods.md | 25 +++++++++++++++++++++++++ docs/concepts/streams.md | 2 +- 6 files changed, 56 insertions(+), 9 deletions(-) diff --git a/.zed/rules b/.zed/rules index f77ed3ea4..11173d2f4 100644 --- a/.zed/rules +++ b/.zed/rules @@ -498,10 +498,11 @@ conversion. Two specializations: Stream (core/streams/arrow_table_stream.py) — immutable (Tag, Data) sequence. Key methods: output_schema(), keys(), iter_data(), as_table(). -Every stream, source and node has run() with no arguments (developer, November 13, 2026, -ITL-1081): it does the work as_table() does (a job node runs as in a job) and returns a +Every stream, source and node has run() with no positional arguments (developer, November 13, +2026, ITL-1081): it does the work as_table() does (a job node runs as in a job) and returns a StreamView of every row; the view's head/tail/sort_by_tags choose what is shown. -head() and the notebook preview never compute. Never give run() arguments. +head() and the notebook preview never compute. A job node's run() takes one keyword-only +argument, load_only (ITL-1095); never give any other run() an argument. Source (core/sources/) — produces a stream from external data. ArrowTableSource is the core implementation; CSV/Delta/DataFrame/Dict/List sources all delegate to it internally. Each @@ -786,6 +787,12 @@ data. The rules: cannot be named (ITL-1043). job.run(load_only=True) computes nothing and writes nothing: function nodes load their own stored entries for this run's inputs and skip (and count) the others, operators run without storing, side-effect nodes pass rows on undelivered. + A job node's run(*, load_only=False) is state-driven: a node that holds a completed, + non-stale run (for a full run: not load-only, nor anything above it) runs nothing; + otherwise its non-current dependencies run first, each once, on the sync orchestrator + (pipeline/node_run.py), and the run commits as job.run() does. A failed end-of-run + commit clears every node of the run. A load-only run's rows have status + loaded_only. Only job nodes' run() takes an argument (load_only). - Empty data → ArrowTableSource raises ValueError("Table is empty"). - DerivedSource over a job node with nothing stored → presents zero rows with the origin's columns (no computed records; no error), in the plain form and with tags_as_data. diff --git a/CLAUDE.md b/CLAUDE.md index 93ff72c09..93de212f3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -513,10 +513,11 @@ conversion. Two specializations: **Stream** (`core/streams/arrow_table_stream.py`) — immutable (Tag, Data) sequence. Key methods: `output_schema()`, `keys()`, `iter_data()`, `as_table()`. -Every stream, source and node has `run()` with no arguments (developer, November 13, 2026, -ITL-1081): it does the work `as_table()` does (a job node runs as in a job) and returns a +Every stream, source and node has `run()` with no positional arguments (developer, November 13, +2026, ITL-1081): it does the work `as_table()` does (a job node runs as in a job) and returns a `StreamView` of every row; the view's `head`/`tail`/`sort_by_tags` choose what is shown. -`head()` and the notebook preview never compute. Never give `run()` arguments. +`head()` and the notebook preview never compute. A job node's `run()` takes one keyword-only +argument, `load_only` (ITL-1095); never give any other `run()` an argument. **Source** (`core/sources/`) — produces a stream from external data. `ArrowTableSource` is the core implementation; CSV/Delta/DataFrame/Dict/List sources all delegate to it internally. Each @@ -827,6 +828,12 @@ data. The rules: cannot be named (ITL-1043). `job.run(load_only=True)` computes nothing and writes nothing: function nodes load their own stored entries for this run's inputs and skip (and count) the others, operators run without storing, side-effect nodes pass rows on undelivered. + A job node's `run(*, load_only=False)` is state-driven: a node that holds a completed, + non-stale run (for a full run: not load-only, nor anything above it) runs nothing; + otherwise its non-current dependencies run first, each once, on the sync orchestrator + (`pipeline/node_run.py`), and the run commits as `job.run()` does. A failed end-of-run + commit clears every node of the run. A load-only run's rows have status + `loaded_only`. Only job nodes' `run()` takes an argument (`load_only`). - Empty data → `ArrowTableSource` raises `ValueError("Table is empty")`. - `DerivedSource` over a job node with nothing stored → presents zero rows with the origin's columns (no computed records; no error), in the plain form and with `tags_as_data`. diff --git a/changelog.d/ITL-1095.md b/changelog.d/ITL-1095.md index 8577bb28c..cc54a29fe 100644 --- a/changelog.d/ITL-1095.md +++ b/changelog.d/ITL-1095.md @@ -3,11 +3,15 @@ - A job node shows the current run; `node.as_source()` shows everything stored for it (developer, November 13, 2026; ITL-1095). Before a run in this process, a function or operator job node is empty: `as_table()`, `iter_data()`, the dataframe methods, `flow()`, `head()` and the notebook preview return zero rows with the node's columns, and `head()` (status `not_computed`) says the node has not run and points to `run()` and to `node.as_source()`. Before, a function node built over a store that already held results, or loaded with `PipelineJob.load`, returned every row stored under its key, and an operator node in `CacheMode.REPLAY` returned every logged run. The same holds after `clear_cache()`, after a run that raised, and while the node is stale. Migration: read stored results with `node.as_source()` (or `get_all_records()`), or run the job first; a run over a store that holds every result computes nothing. - A function or operator job node that has not run in this process is stale (`is_stale` is `True`; it has no modified time), and a job node is stale also when any function or operator job node it depends on is. Binding a source to a job's source slot (`job.bind(sources=...)`) marks the slot modified, so every node fed by it is stale and empty until it runs again. - A loaded operator node no longer raises from `as_table()` or `iter_data()`: it is empty, like every node that has not run. Its stored view (`as_source()`, `get_all_records()`) raises instead when the job file does not record its run's content hash and the store holds nothing under the hash the loaded job computes (ITL-1043). +- A job node's `run()` is state-driven and runs what the node depends on (developer, November 13, 2026; ITL-1095). A node that holds a completed run and is not stale runs nothing: `run()` returns the rows it holds. Otherwise every job node it depends on that holds no current run runs first, each once, as `job.run()` would run that part of the job, then the node; the run's writes are committed when it ends (before, `node.run()` read an upstream node's stored rows, or nothing, and left its last pipeline record and a `LOG` operator's output uncommitted). `run()` of a node whose function pod is a placeholder, or that was loaded without one, raises unless `load_only=True`; a `CACHE_ONLY` node's `run()` raises (its input is unavailable): read its stored results with `as_source()`. A job node's `run()` takes one keyword-only argument, `load_only`; every other stream's `run()` still takes none. +- If the commit at the end of `job.run()` or `node.run()` fails, every function and operator node of the run is cleared (not run, stale) before the error propagates. ### Added - `job.run(load_only=True)` and `await job.run_async(load_only=True)` fill a job's nodes with what is already stored for this run's inputs and compute nothing (ITL-1095): sources are read, operators run on the rows that reach them (a `CacheMode.LOG` cache writes nothing), each function node takes the stored results of its inputs and skips the inputs without one, a side-effect node passes its rows on without delivering anything, and nothing is written to the store. `RunSummary.load_only`, `NodeRunSummary.not_stored` (inputs skipped) and `NodeRunSummary.inputs_incomplete` (the function nodes above a node that skipped inputs) report it; loaded rows count in `reused`, and the summary's text says "3 loaded, 1 not stored (skipped)". +- `node.run(load_only=True)`, and `node.last_run`, the `RunSummary` of a node's last `run()`. `ViewStatus.LOADED_ONLY`: the `head()` and `run()` view of a node filled by a load-only run, captioned "N loaded, M inputs without a stored result" and naming the function nodes above it that skipped inputs. `holds_current_run(load_only=...)` on function and operator job nodes says whether `run()` would run them. + ### Fixed - On the asynchronous orchestrator, an operator node in `CacheMode.REPLAY` stamps itself after its inputs close, so the rows it replayed are not judged out of date by the time the run ends. diff --git a/design_issues/ITL-1095-CR1.md b/design_issues/ITL-1095-CR1.md index b2e853b2c..ff78f3191 100644 --- a/design_issues/ITL-1095-CR1.md +++ b/design_issues/ITL-1095-CR1.md @@ -20,5 +20,9 @@ shows everything stored. Function and operator job nodes read empty (typed zero `clear_cache()`, after a run that raised and while stale; no read touches the store. A never-run or cleared node has no modified time (stale); `core/nodes/staleness.py` makes a job node stale when any function or operator job node above it is; a `SourceJobNode` is modified when bound or -when its bound source is. Planned in the same issue: `run(load_only=True)` and the `node.run()` -cascade. +when its bound source is. `job.run(load_only=True)` fills the nodes from what is stored and +computes and writes nothing. A job node's `run(*, load_only=False)` is state-driven: a current +node runs nothing; otherwise its non-current dependencies run first, once each, on the sync +orchestrator restricted to them (`pipeline/node_run.py`), and the run commits as `job.run()` +does; a failed end-of-run commit clears every node of the run. A load-only run's rows have +status `loaded_only` and are never current for a full run. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index cbb4a4415..2492f6349 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -578,6 +578,31 @@ incomplete: a group formed after it may lack members, and the result stored for group by an earlier run may then be loaded for it. `run_async(load_only=True)` does the same on the asynchronous orchestrator. +### Running one node: `node.run()` + +`job.nodes[label].run()` runs that node as `job.run()` would, together with the job nodes +it depends on, and returns a view of its rows. What runs depends on each node's state +(developer, November 13, 2026): + +| State of a node | What it shows | `run()` | `run(load_only=True)` | +|---|---|---|---| +| Not run in this process (a fresh or loaded job), cleared, or stale | nothing: `head()` says "not run" (`not_computed`) | runs it | loads it | +| Filled by a load-only run | the rows it loaded: status `loaded_only`, "N loaded, M inputs without a stored result" | runs it, computing what it skipped | nothing: shows what it holds | +| Filled by a run (`job.run()` or `run()`) | that run's rows (`rows`) | nothing: shows what it holds | nothing: shows what it holds | + +A node is stale when anything it depends on changed after its run: an upstream node ran +again or was cleared, or a source was bound again (`job.bind(sources=...)`). Before a node +runs, every job node it depends on that is not current for that kind of run runs first, +each once, in order; the others feed it their rows. A load-only run's rows are never current +for a full run, nor are the rows of any node below them. `node.last_run` holds the counts, +as `job.last_run` does for `job.run()`, and the run's writes are committed when it ends. +Side effects above the node are delivered as `job.run()` delivers them; a node that holds a +current run is not run again, so to deliver a side effect again (with +`track_completion=False`), call `clear_cache()` on its node first. Staleness comes from +modified times only: a source whose rows change without its object changing (a table read +lazily, a file rewritten in place) is not seen. Build a new source object and bind it, or +clear the nodes, to run them again. + ### When results are committed Inside `job.run()`, each function node commits its results and pipeline records to the diff --git a/docs/concepts/streams.md b/docs/concepts/streams.md index 838cc31fe..7d61ad3ba 100644 --- a/docs/concepts/streams.md +++ b/docs/concepts/streams.md @@ -205,7 +205,7 @@ What `run()` does on each kind of object: | A `PollingSource` | polls, as `as_table()` does | | A `CachedSource` | reads its cache and writes the wrapped source's new rows to it | | A function pod's or operator's output stream | computes what is not computed yet | -| A job node | runs the node as a job run would (`as_table()` of a job node only reads), then returns the view of that run's rows | +| A job node | runs the node as a job run would, after every job node it depends on that holds no current run (each once), commits, and returns the view of that run's rows; a node that holds a current run (completed in this process and not stale) runs nothing. `run(load_only=True)` loads stored results only. `as_table()` of a job node only reads | | A side-effect stream | delivers its side effects for every row, on every call, and returns the rows it passed on; `head()` still shows none | | A pipeline blueprint node, an unbound input slot | raises, as `as_table()` does | From a9d07f06a5eae40e48df03adbe1ddf871475152a Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 22:08:07 -0800 Subject: [PATCH 20/48] fix(function_node): any raise of a run clears the node's rows, also a refusal or an observer failure (ITL-1093) Addresses the second CodeRabbit review of PR #378: a sync run refused for its input's data context, or failing in on_node_end, kept the previous run's rows. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/function_node.py | 28 ++++++++----- .../test_job_node_rows_after_run.py | 40 +++++++++++++++++++ .../test_pipeline/test_stored_data_context.py | 32 +++++++++++++++ 3 files changed, 91 insertions(+), 9 deletions(-) diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 252c98c6e..1f81a2f34 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -1940,11 +1940,17 @@ def execute( if self._function_pod is None: raise self._cannot_compute_error() - self._begin_run() - # A whole-input condition, not a row failure: the error policy does not apply. - require_same_data_context( - input_stream.data_context_key, self._stored_data_context_key - ) + try: + self._begin_run() + # A whole-input condition, not a row failure: the error policy does not + # apply. + require_same_data_context( + input_stream.data_context_key, self._stored_data_context_key + ) + except BaseException: + # A run that raised defines no rows, also when it was refused at once. + self.clear_cache() + raise node_label = self.label node_hash = self.content_hash().to_string() @@ -2041,6 +2047,9 @@ def execute( self._commit_or_end_node( self._commit_at_node_end, ctx_obs, node_label, node_hash ) + # The node's rows are those of this run's inputs, whatever else it caches. + self._set_run_rows(emitted_entry_ids) + ctx_obs.on_node_end(node_label, node_hash) except Exception as run_error: self._commit_after_failure(run_error) # A run that raised defines no rows: the node reads as one with no run. @@ -2050,9 +2059,6 @@ def execute( self.clear_cache() raise - # The node's rows are those of this run's inputs, whatever else it caches. - self._set_run_rows(emitted_entry_ids) - ctx_obs.on_node_end(node_label, node_hash) # Mark this node as freshly computed so subsequent iter_data() calls # skip the is_stale check and serve results directly from the in-memory cache. self._update_modified_time() @@ -3389,7 +3395,9 @@ async def async_execute( try: self._begin_run() except StoreError: - # Nothing was delivered; downstream must not wait for rows. + # Nothing was delivered; downstream must not wait for rows. A run that + # raised defines no rows. + self.clear_cache() await output.close() raise status = self.load_status @@ -3414,6 +3422,8 @@ async def async_execute( self._input_stream.data_context_key, self._stored_data_context_key ) except InputValidationError: + # A run that raised defines no rows, also when it was refused at once. + self.clear_cache() await output.close() raise node_label = self.label diff --git a/tests/test_pipeline/test_job_node_rows_after_run.py b/tests/test_pipeline/test_job_node_rows_after_run.py index e1a53ee7e..e45ff97bc 100644 --- a/tests/test_pipeline/test_job_node_rows_after_run.py +++ b/tests/test_pipeline/test_job_node_rows_after_run.py @@ -435,3 +435,43 @@ def test_async_one_of_two_equal_inputs_fails(self) -> None: received = self._run_async(node) assert received == [(0, 6), (2, 8)] assert self._held(node) == received + + +class TestRefusedOrFailedRunAfterAGoodRun: + """A run that raises after a good run drops that run's rows too (ITL-1093). + + The node then reads as a node with no run in this process: every row stored under + its content hash, read from the store, whatever raised. An input refused for its data + context is tested in ``test_stored_data_context.py``, which registers a second one. + """ + + @staticmethod + def _job(db: Any) -> PipelineJob: + """Return a job applying ``inc`` to a source of 6 rows as ``node``.""" + return _single(inc.pod, ROWS, db) + + @pytest.mark.parametrize("kind", ORCHESTRATORS) + def test_observer_failure_at_node_end_drops_the_rows(self, kind: str) -> None: + """An observer raising at the node's end leaves the node with no run's rows.""" + from orcapod.pipeline.observer import NoOpObserver + + class FailsAtNodeEnd(NoOpObserver): + """An observer whose ``on_node_end`` raises.""" + + def on_node_end(self, node_label: str, node_hash: str) -> None: + """Raise at the function node's end, as a broken observer would.""" + if node_label == "node": + raise RuntimeError("observer failed") + + db = InMemoryArrowDatabase() + job = self._job(db) + job.run(orchestrator=_orchestrator(kind)) + job.bind(sources={"s": _source(ROWS[:3])}) + job.run(orchestrator=_orchestrator(kind)) + node = job.nodes["node"] + assert _values(node, lambda r: r["k"]) == [0, 1, 2] + # The async orchestrator raises its task group's error around it. + with pytest.raises((RuntimeError, ExceptionGroup)): + job.run(orchestrator=_orchestrator(kind), observer=FailsAtNodeEnd()) + # Every stored row (the 6-row run's), read from the store. + assert _values(node, lambda r: r["k"]) == [0, 1, 2, 3, 4, 5] diff --git a/tests/test_pipeline/test_stored_data_context.py b/tests/test_pipeline/test_stored_data_context.py index 49708925a..17f072b73 100644 --- a/tests/test_pipeline/test_stored_data_context.py +++ b/tests/test_pipeline/test_stored_data_context.py @@ -1254,6 +1254,38 @@ def test_function_node_raises_and_writes_nothing(self, store: Store): assert calls == [] assert reopened(store).list_sources() == [] + def test_refused_input_after_a_good_run_leaves_no_run_rows(self, store: Store): + """A run refused for its input's data context defines no rows (ITL-1093). + + After a good run the node holds that run's rows; a later sync run refused for its + input's data context drops them, and the node reads its stored rows, as with no + run in this process. (On the async path the context checked is the node's own + input stream's, so a node refused there never had a run.) + """ + table = pa.table( + {"k": ["a", "b"], "x": [1, 2]}, + schema=pa.schema( + [ + pa.field("k", pa.large_string(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] + ), + ) + node = FunctionJobNode( + function_pod=double.pod, # pyright: ignore[reportArgumentType] # the decorator types .pod as the pod base class + input_stream=ArrowTableStream(table, tag_columns=["k"]), + pipeline_database=store, + result_database=store, + ) + node.run() + assert node.head(10).origin == "held" + + with pytest.raises(InputValidationError): + node.execute(_other_context_stream()) + + assert node.head(10).origin == "stored" + assert sorted(row["double"] for row in node.as_table().to_pylist()) == [2, 4] + def test_add_pipeline_record_refuses_an_input_of_another_data_context( self, store: Store ): From 2930456789de16251ee464d4f3b6cdb2b0c4dc58 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 22:10:18 -0800 Subject: [PATCH 21/48] test(function_node): a CACHE_ONLY node's run() raises and points to as_source() (ITL-1095) Co-Authored-By: Claude Opus 5.5 --- tests/test_core/nodes/test_function_node_iteration.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_core/nodes/test_function_node_iteration.py b/tests/test_core/nodes/test_function_node_iteration.py index e28d54b4d..10a5266a9 100644 --- a/tests/test_core/nodes/test_function_node_iteration.py +++ b/tests/test_core/nodes/test_function_node_iteration.py @@ -123,10 +123,10 @@ def test_as_table_fresh_node_returns_empty_no_compute(self): assert "id" in table.column_names assert "result" in table.column_names - def test_run_cache_only_is_noop(self): - """run() on a CACHE_ONLY node computes nothing and returns the stored rows' view.""" + def test_run_cache_only_raises_and_points_to_the_stored_view(self): + """run() on a CACHE_ONLY node computes nothing: its input is unavailable, so it + has nothing to run or look up, and it says to read ``as_source()`` (ITL-1095).""" from orcapod.pipeline.serialization import LoadStatus - from orcapod.streams import StreamView node = _make_node() # A loaded node whose upstream is unavailable keeps that input stream; only its @@ -134,9 +134,10 @@ def test_run_cache_only_is_noop(self): node._load_status = LoadStatus.CACHE_ONLY with patch.object(node, "execute") as mock_exec: - view = node.run() + for load_only in (False, True): + with pytest.raises(RuntimeError, match=r"as_source\(\)"): + node.run(load_only=load_only) mock_exec.assert_not_called() - assert isinstance(view, StreamView) def test_run_unavailable_raises(self): """run() on an UNAVAILABLE node raises RuntimeError.""" From b693b50e336ee66e1b193649c0500c593ece43c3 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 22:11:12 -0800 Subject: [PATCH 22/48] test: #378's new raised-run tests read the stored view (ITL-1095); load-only path clears on a refused run Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/function_node.py | 14 +++++++------- .../test_pipeline/test_job_node_rows_after_run.py | 6 ++++-- tests/test_pipeline/test_stored_data_context.py | 14 +++++++++----- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index d48ff8b42..2d4513dab 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -1627,21 +1627,21 @@ def _execute_load_only( from orcapod.pipeline.observer import NoOpObserver from orcapod.pipeline.serialization import LoadStatus - if self.load_status == LoadStatus.UNAVAILABLE: - raise self._no_store_error() - require_same_data_context( - input_stream.data_context_key, self._stored_data_context_key - ) node_label = self.label node_hash = self.content_hash().to_string() obs = observer if observer is not None else NoOpObserver() ctx_obs = obs.contextualize(*self.node_identity_path) - tag_schema = input_stream.output_schema(columns={"system_tags": True})[0] - ctx_obs.on_node_start(node_label, node_hash, tag_schema=tag_schema) output: list[tuple[TagProtocol, DataProtocol]] = [] loaded_ids: list[bytes] = [] not_stored = 0 try: + if self.load_status == LoadStatus.UNAVAILABLE: + raise self._no_store_error() + require_same_data_context( + input_stream.data_context_key, self._stored_data_context_key + ) + tag_schema = input_stream.output_schema(columns={"system_tags": True})[0] + ctx_obs.on_node_start(node_label, node_hash, tag_schema=tag_schema) inputs = [ (tag, data, self.compute_base_entry_id(tag, data)) for tag, data in input_stream.iter_data() diff --git a/tests/test_pipeline/test_job_node_rows_after_run.py b/tests/test_pipeline/test_job_node_rows_after_run.py index b98c66655..6634a4fdb 100644 --- a/tests/test_pipeline/test_job_node_rows_after_run.py +++ b/tests/test_pipeline/test_job_node_rows_after_run.py @@ -467,5 +467,7 @@ def on_node_end(self, node_label: str, node_hash: str) -> None: # The async orchestrator raises its task group's error around it. with pytest.raises((RuntimeError, ExceptionGroup)): job.run(orchestrator=_orchestrator(kind), observer=FailsAtNodeEnd()) - # Every stored row (the 6-row run's), read from the store. - assert _values(node, lambda r: r["k"]) == [0, 1, 2, 3, 4, 5] + # The node holds no run (it reads empty, ITL-1095); the store keeps every row. + assert _values(node, lambda r: r["k"]) == [] + stored = node.as_source().as_table().to_pylist() + assert sorted(r["k"] for r in stored) == [0, 1, 2, 3, 4, 5] diff --git a/tests/test_pipeline/test_stored_data_context.py b/tests/test_pipeline/test_stored_data_context.py index 79b2356ab..a3d678727 100644 --- a/tests/test_pipeline/test_stored_data_context.py +++ b/tests/test_pipeline/test_stored_data_context.py @@ -48,6 +48,7 @@ from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator from orcapod.side_effects import SideEffectPod, sink_pod from orcapod.sources import ArrowTableSource, DictSource +from orcapod.core.streams import ViewStatus from orcapod.types import CacheMode, NodeConfig Orchestrator = SyncPipelineOrchestrator | AsyncPipelineOrchestrator @@ -1266,9 +1267,10 @@ def test_refused_input_after_a_good_run_leaves_no_run_rows(self, store: Store): """A run refused for its input's data context defines no rows (ITL-1093). After a good run the node holds that run's rows; a later sync run refused for its - input's data context drops them, and the node reads its stored rows, as with no - run in this process. (On the async path the context checked is the node's own - input stream's, so a node refused there never had a run.) + input's data context drops them, and the node reads as with no run in this + process: empty (ITL-1095); its stored view keeps the rows. (On the async path the + context checked is the node's own input stream's, so a node refused there never + had a run.) """ table = pa.table( {"k": ["a", "b"], "x": [1, 2]}, @@ -1291,8 +1293,10 @@ def test_refused_input_after_a_good_run_leaves_no_run_rows(self, store: Store): with pytest.raises(InputValidationError): node.execute(_other_context_stream()) - assert node.head(10).origin == "stored" - assert sorted(row["double"] for row in node.as_table().to_pylist()) == [2, 4] + assert node.head(10).status is ViewStatus.NOT_COMPUTED + assert node.as_table().num_rows == 0 + stored = node.as_source().as_table().to_pylist() + assert sorted(row["double"] for row in stored) == [2, 4] def test_add_pipeline_record_refuses_an_input_of_another_data_context( self, store: Store From 3610fd20d59c05a0e78f4554f3d378999eb71231 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 22:13:47 -0800 Subject: [PATCH 23/48] feat(pipeline): a loaded job is filled from its store by run(load_only=True) (ITL-1095) loaded.run(load_only=True) and a loaded node's run(load_only=True) read the job's sources (rebuilt or bound), run the rebuilt operators (a loaded REPLAY operator replays) and give each function node the results its store holds for those inputs; nodes below an unbound source stay empty. loaded.run() still raises, now naming run(load_only=True) and as_source(). The four loaded-job tests that waited for this stage read through a load-only run again (no xfails left). Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 1 + CLAUDE.md | 1 + changelog.d/ITL-1095.md | 2 + docs/concepts/function-pods.md | 21 ++- src/orcapod/core/nodes/function_node.py | 4 +- src/orcapod/core/nodes/operator_node.py | 10 +- src/orcapod/pipeline/job.py | 32 ++-- .../loaded_job/test_loaded_job_readback.py | 101 ++++++----- .../loaded_job/test_loaded_load_only.py | 166 ++++++++++++++++++ 9 files changed, 276 insertions(+), 62 deletions(-) create mode 100644 tests/test_pipeline/loaded_job/test_loaded_load_only.py diff --git a/.zed/rules b/.zed/rules index 11173d2f4..94beea7a3 100644 --- a/.zed/rules +++ b/.zed/rules @@ -793,6 +793,7 @@ data. The rules: (pipeline/node_run.py), and the run commits as job.run() does. A failed end-of-run commit clears every node of the run. A load-only run's rows have status loaded_only. Only job nodes' run() takes an argument (load_only). + A loaded job (PipelineJob.load) runs only with load_only=True. - Empty data → ArrowTableSource raises ValueError("Table is empty"). - DerivedSource over a job node with nothing stored → presents zero rows with the origin's columns (no computed records; no error), in the plain form and with tags_as_data. diff --git a/CLAUDE.md b/CLAUDE.md index 93de212f3..4025641ff 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -834,6 +834,7 @@ data. The rules: (`pipeline/node_run.py`), and the run commits as `job.run()` does. A failed end-of-run commit clears every node of the run. A load-only run's rows have status `loaded_only`. Only job nodes' `run()` takes an argument (`load_only`). + A loaded job (`PipelineJob.load`) runs only with `load_only=True`. - Empty data → `ArrowTableSource` raises `ValueError("Table is empty")`. - `DerivedSource` over a job node with nothing stored → presents zero rows with the origin's columns (no computed records; no error), in the plain form and with `tags_as_data`. diff --git a/changelog.d/ITL-1095.md b/changelog.d/ITL-1095.md index cc54a29fe..543769b5f 100644 --- a/changelog.d/ITL-1095.md +++ b/changelog.d/ITL-1095.md @@ -12,6 +12,8 @@ - `node.run(load_only=True)`, and `node.last_run`, the `RunSummary` of a node's last `run()`. `ViewStatus.LOADED_ONLY`: the `head()` and `run()` view of a node filled by a load-only run, captioned "N loaded, M inputs without a stored result" and naming the function nodes above it that skipped inputs. `holds_current_run(load_only=...)` on function and operator job nodes says whether `run()` would run them. +- A job loaded with `PipelineJob.load` can be filled from its store: `loaded.run(load_only=True)` (and `node.run(load_only=True)` of a loaded node) reads the job's sources (rebuilt from files, or bound with `loaded.bind(sources=...)`), runs the rebuilt operators and gives each function node the results its store holds for those inputs; nodes below an unbound source stay empty. `loaded.run()` without `load_only` still raises, and says how to load. + ### Fixed - On the asynchronous orchestrator, an operator node in `CacheMode.REPLAY` stamps itself after its inputs close, so the rows it replayed are not judged out of date by the time the run ends. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 2492f6349..785369fd0 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -765,10 +765,23 @@ print(loaded.nodes["bmi"].as_source().as_table().sort_by("subject_id").to_pandas # 2 mouse_03 2297.520661 ``` -A loaded job is for reading stored results, through each node's stored view, -`node.as_source()`: its nodes have not run in this process, so they are empty. It cannot be -run again: function pods are not saved with the job, so `loaded.run()` raises. To compute -again, build the job from your code as before. +A loaded job is for reading stored results. Its nodes have not run in this process, so +they are empty; each node's stored view, `node.as_source()`, reads what the store holds. A +loaded job cannot compute: function pods are not saved with it, so `loaded.run()` raises. +`loaded.run(load_only=True)` fills its nodes from the store instead, as for any job (see +*Loading without computing* above): the sources are read (an in-memory source must be bound +first, since its data is not saved), the rebuilt operators run, and each function node takes +the results stored for those inputs. Nodes below a source that is not available stay empty. +`node.run(load_only=True)` does the same for one node and what it depends on. + +```python +loaded.bind(sources={"mice": DictSource(data=mice, tag_columns=["subject_id"], source_id="mice")}) +summary = loaded.run(load_only=True) +print(loaded.nodes["bmi"].as_table().num_rows, summary.nodes["bmi"].not_stored) +# 3 0 +``` + +To compute again, build the job from your code as before. The store is the source of truth: a loaded job's stored views show what the store holds for it. The job file names each run's rows by the content hash the run stored them under, so a diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 2d4513dab..39aca8292 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -1778,8 +1778,8 @@ def _cannot_compute_error(self) -> RuntimeError: """ return RuntimeError( f"FunctionJobNode {self.label!r} was loaded by PipelineJob.load and has no " - "function pod, so it cannot compute; read its stored results with " - "as_source()." + "function pod, so it cannot compute; fill it from the store with " + "run(load_only=True), or read its stored results with as_source()." ) def _stored_record_count(self) -> int: diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index a9aa8ec30..9725f928b 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -1018,7 +1018,9 @@ def _cannot_compute_error(self) -> RuntimeError: """Describe why this loaded node cannot execute an operator.""" return RuntimeError( f"OperatorJobNode {self.label!r} was loaded by PipelineJob.load and " - "cannot compute; create a fresh PipelineJob to run the pipeline again." + "cannot compute; fill it from the store with run(load_only=True), read " + "its stored results with as_source(), or create a fresh PipelineJob to " + "run the pipeline again." ) def _filter_by_content_hash(self, table: pa.Table) -> pa.Table: @@ -1341,7 +1343,8 @@ def execute( """ from orcapod.pipeline.observer import NoOpObserver - if self._loaded: + # A loaded node runs only in a load-only run, with its rebuilt operator. + if self._loaded and (not load_only or self._operator is None): raise self._cannot_compute_error() node_label = self.label @@ -1753,7 +1756,8 @@ async def async_execute( """ from orcapod.pipeline.observer import NoOpObserver - if self._loaded: + # A loaded node runs only in a load-only run, with its rebuilt operator. + if self._loaded and (not load_only or self._operator is None): await output.close() raise self._cannot_compute_error() diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index e36f6ca08..91644b9a5 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -929,7 +929,7 @@ def run( """ from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator - prepared = self._prepare_run(commit_every_rows) + prepared = self._prepare_run(commit_every_rows, load_only=load_only) counter = RunCountingObserver() effective_orchestrator = orchestrator or SyncPipelineOrchestrator() with self._executing(prepared, commit_every_rows): @@ -992,7 +992,7 @@ async def run_async( f"{type(orchestrator).__name__}. Use job.run(orchestrator=...) to run " "it." ) - prepared = self._prepare_run(commit_every_rows) + prepared = self._prepare_run(commit_every_rows, load_only=load_only) counter = RunCountingObserver() with self._executing(prepared, commit_every_rows): _ = await orchestrator.run_async( @@ -1008,11 +1008,14 @@ async def run_async( warn_row_failures(summary, sys._getframe(1)) # pyright: ignore[reportPrivateUsage] # the caller's line is where the warning is shown return summary - def _prepare_run(self, commit_every_rows: int) -> "_PreparedRun": + def _prepare_run( + self, commit_every_rows: int, *, load_only: bool = False + ) -> "_PreparedRun": """Check the job can run and build what a run needs; shared by both run forms. Args: commit_every_rows: As for ``run``. + load_only: As for ``run``: a loaded job can run only this way. Returns: The execution DAG, run id, pipeline URI and the store views of the run. @@ -1039,14 +1042,17 @@ def _prepare_run(self, commit_every_rows: int) -> "_PreparedRun": "PipelineJob has no compiled pipeline. " "Record invocations inside a 'with job:' block first." ) - # A loaded job reads the results its saved run stored and never runs: - # function pods are not saved with it, and even an operator-only loaded - # job, whose operators are rebuilt, stays read-only. - if self._loaded: + # A loaded job never computes: function pods are not saved with it, and even + # an operator-only loaded job, whose operators are rebuilt, stays read-only. + # A load-only run fills it from the store (ITL-1095). + if self._loaded and not load_only: raise RuntimeError( - "PipelineJob.run() cannot execute a loaded job: PipelineJob.load() " - "is for reading stored results only. To run a new computation, " - "create a fresh PipelineJob from the original pipeline definition." + "PipelineJob.run() cannot execute a loaded job: its function pods are " + "not saved with it, so it cannot compute. Fill its nodes from the " + "store with run(load_only=True) (bind in-memory sources first), or " + "read a node's stored results with node.as_source(). To run a new " + "computation, create a fresh PipelineJob from the original pipeline " + "definition." ) # Until this run finishes, what the store holds for a node is not certain. self._last_run_finished = False @@ -1087,7 +1093,11 @@ def _prepare_run(self, commit_every_rows: int) -> "_PreparedRun": isinstance(node, (FunctionJobNode, OperatorJobNode)) and node.load_status == LoadStatus.UNAVAILABLE ) - or (isinstance(node, FunctionJobNode) and node.producer is None) + or ( + not load_only + and isinstance(node, FunctionJobNode) + and node.producer is None + ) ] if unavailable_labels: raise RuntimeError( diff --git a/tests/test_pipeline/loaded_job/test_loaded_job_readback.py b/tests/test_pipeline/loaded_job/test_loaded_job_readback.py index b14dff36e..53bd101b5 100644 --- a/tests/test_pipeline/loaded_job/test_loaded_job_readback.py +++ b/tests/test_pipeline/loaded_job/test_loaded_job_readback.py @@ -69,6 +69,39 @@ def _rewrite_job_file( return str(path) +def _loaded_and_filled(saved: pathlib.Path) -> PipelineJob: + """Load the saved job, bind its in-memory sources and fill it from the store. + + A loaded node is empty until a load-only run fills it (ITL-1095). The sources bound + are those of ``save_job_script.py``; the CSV and Delta sources are rebuilt. + """ + from orcapod.sources import ArrowTableSource, ListSource + + loaded = _load(saved) + arrow = ArrowTableSource( + pa.table( + {"id": [0, 1], "y": [3, 4]}, + schema=pa.schema( + [ + pa.field("id", pa.int64(), nullable=False), + pa.field("y", pa.int64(), nullable=False), + ] + ), + ), + tag_columns=["id"], + source_id="asrc", + ) + loaded.bind( + sources={ + "dsrc": _dict_source(), + "asrc": arrow, + "lsrc": ListSource("z", [1, 2], source_id="lsrc"), + } + ) + _ = loaded.run(load_only=True) + return loaded + + def _dict_source() -> DictSource: """Return a two-row in-memory source with tag ``id`` and data ``x``.""" return DictSource( @@ -235,22 +268,18 @@ def test_loaded_function_node_as_table_returns_stored_rows( assert _sorted(table, TAG_OF[label]).equals(_sorted(expected, TAG_OF[label])) -@pytest.mark.xfail( - strict=True, - reason="ITL-1095: a loaded node reads empty until loaded.run(load_only=True), " - "which reads its entries this way (stage 4)", -) @pytest.mark.parametrize("label", FUNCTION_NODES) def test_loaded_function_node_as_table_all_info_has_stream_columns( saved: pathlib.Path, label: str ) -> None: - """With ``all_info``, a loaded node's table has every live column and more.""" + """With ``all_info``, a loaded node's table, once a load-only run has filled it, + has every live column and more.""" # The loaded table follows the documented stream semantics: every column of the # live table, with the same values and types, plus the data context and the # source-info columns. (The live FunctionJobNode.as_table() drops those two # groups -- a separate, pre-existing bug the loaded node does not copy.) expected = pq.read_table(saved / "expected" / f"{label}.all_info.parquet") - table = _load(saved).nodes[label].as_source().as_table(all_info=True) + table = _loaded_and_filled(saved).nodes[label].as_table(all_info=True) data_column = next(iter(_load(saved).nodes[label].output_schema()[1])) assert "_context_key" in table.column_names assert f"_source_{data_column}" in table.column_names @@ -388,15 +417,11 @@ def test_other_context_records_do_not_satisfy_saved_expectation( assert rows is None or len(rows) == 0 -@pytest.mark.xfail( - strict=True, - reason="ITL-1095: a loaded node reads empty until loaded.run(load_only=True), " - "which reads its entries this way (stage 4)", -) def test_recomputed_entry_is_read_once_with_its_latest_result( tmp_path: pathlib.Path, ) -> None: - """A loaded node reads one row per entry: the greatest recomputation index wins.""" + """A loaded node's load-only run reads one row per entry: the greatest recomputation + index wins.""" path = _save_small_job(tmp_path, _record_double) node = PipelineJob.load(path).nodes["dbl"] # A loaded node is empty until it runs (ITL-1095): read its stored view. @@ -405,7 +430,10 @@ def test_recomputed_entry_is_read_once_with_its_latest_result( _append_recomputed_record(node, entry_id, recomputation_index=2, value=222) _append_recomputed_record(node, entry_id, recomputation_index=1, value=111) - reloaded = PipelineJob.load(path).nodes["dbl"].as_source() + reloaded_job = PipelineJob.load(path) + reloaded_job.bind(sources={"dsrc": _dict_source()}) + _ = reloaded_job.run(load_only=True) + reloaded = reloaded_job.nodes["dbl"] table = reloaded.as_table() assert table.num_rows == 2 rows = {row["id"]: row["dbl"] for row in table.to_pylist()} @@ -442,15 +470,11 @@ def test_live_node_hot_load_reads_the_latest_recomputed_result( assert values[entry_id] == 222 -@pytest.mark.xfail( - strict=True, - reason="ITL-1095: a loaded node reads empty until loaded.run(load_only=True), " - "which reads its entries this way (stage 4)", -) def test_loaded_node_with_ephemeral_results_does_not_crash( tmp_path: pathlib.Path, ) -> None: - """Ephemeral results come back as placeholders; the table says why it has none.""" + """Results kept in an ephemeral store are gone after the run: a load-only run of the + loaded job counts them as not stored and shows no rows; it does not crash.""" @function_pod(output_keys="dbl") def double(x: int) -> int: @@ -468,25 +492,14 @@ def make_ephemeral(job: PipelineJob) -> None: node.set_ephemeral_store(InMemoryArrowDatabase()) path = _save_small_job(tmp_path, record, configure=make_ephemeral) - node = PipelineJob.load(path).nodes["dbl"] - assert [type(data).__name__ for _, data in node.iter_data()] == [ - "EmptyData", - "EmptyData", - ] - with pytest.raises(RuntimeError) as excinfo: - _ = node.as_table() - assert str(excinfo.value) == ( - "FunctionJobNode 'dbl': the saved run kept 2 of its results in an ephemeral " - "store, which is not saved with the job, so they cannot be read into a table. " - "iter_data() yields them as EmptyData placeholders." - ) + loaded = PipelineJob.load(path) + loaded.bind(sources={"dsrc": _dict_source()}) + summary = loaded.run(load_only=True) + assert summary.nodes["dbl"].not_stored == 2 + assert summary.nodes["dbl"].reused == 0 + assert loaded.nodes["dbl"].as_table().num_rows == 0 -@pytest.mark.xfail( - strict=True, - reason="ITL-1095: a loaded node reads empty until loaded.run(load_only=True), " - "which reads its entries this way (stage 4)", -) def test_newer_unavailable_result_does_not_expose_old_persistent_result( tmp_path: pathlib.Path, ) -> None: @@ -509,10 +522,13 @@ def test_newer_unavailable_result_does_not_expose_old_persistent_result( records = records.set_column(index, field, pa.array(values, type=field.type)) store.add_records(table_path, records, record_id_column="__rid", flush=True) - node = PipelineJob.load(path).nodes["dbl"] - assert [type(data).__name__ for _, data in node.iter_data()] == ["EmptyData"] * 2 - with pytest.raises(RuntimeError, match="ephemeral store"): - node.as_table() + # The newest result of each entry is gone with its ephemeral store: a load-only run + # counts the entry as not stored and never shows the older persistent result. + loaded = PipelineJob.load(path) + loaded.bind(sources={"dsrc": _dict_source()}) + summary = loaded.run(load_only=True) + assert summary.nodes["dbl"].not_stored == 2 + assert loaded.nodes["dbl"].as_table().num_rows == 0 # --------------------------------------------------------------------------- @@ -801,7 +817,8 @@ def test_loaded_function_node_cannot_execute(saved: pathlib.Path) -> None: _ = job.nodes["dbl"].execute(job.nodes["dsrc"]) assert str(excinfo.value) == ( "FunctionJobNode 'dbl' was loaded by PipelineJob.load and has no function " - "pod, so it cannot compute; read its stored results with as_source()." + "pod, so it cannot compute; fill it from the store with run(load_only=True), " + "or read its stored results with as_source()." ) diff --git a/tests/test_pipeline/loaded_job/test_loaded_load_only.py b/tests/test_pipeline/loaded_job/test_loaded_load_only.py new file mode 100644 index 000000000..a313ebe7f --- /dev/null +++ b/tests/test_pipeline/loaded_job/test_loaded_load_only.py @@ -0,0 +1,166 @@ +"""A loaded job is empty until ``loaded.run(load_only=True)`` fills it (ITL-1095). + +Developer, November 13, 2026: "the initially loaded job is empty", and a load-only run +fills the nodes from what is stored. A loaded job has no function pods, so it can never +compute; with its sources available (rebuilt from files, or bound), a load-only run reads +them, runs the rebuilt operators, and gives each function node the results its store +holds for those inputs. ``loaded.run()`` without ``load_only`` still raises. +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from typing import Any + +import pyarrow as pa +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.core.operators import Join +from orcapod.core.streams import ViewStatus +from orcapod.databases import InMemoryArrowDatabase +from orcapod.sources import ArrowTableSource, CSVSource, DictSource +from orcapod.types import CacheMode + +JOB = "loaded_load_only" + + +@function_pod(output_keys="dbl") +def double(x: int) -> int: + """Double *x*.""" + return 2 * x + + +@function_pod(output_keys="cc") +def increment(c: int) -> int: + """Add one to *c*.""" + return c + 1 + + +def _dict_source() -> DictSource: + return DictSource( + [{"id": 0, "x": 5}, {"id": 1, "x": 10}], tag_columns=["id"], source_id="dsrc" + ) + + +def _arrow_source() -> ArrowTableSource: + table = pa.table( + {"id": [0, 1], "y": [3, 4]}, + schema=pa.schema( + [ + pa.field("id", pa.int64(), nullable=False), + pa.field("y", pa.int64(), nullable=False), + ] + ), + ) + return ArrowTableSource(table, tag_columns=["id"], source_id="asrc") + + +def _save(tmp_path: Path, db: Any, cache_mode: CacheMode | None = None) -> Path: + """Run and save a job: ``dbl`` over an in-memory source joined with another one + (``joined``), and ``inc`` over a CSV source.""" + csv = tmp_path / "c.csv" + csv.write_text("id,c\n0,7\n1,8\n") + job = PipelineJob(name=JOB, database=db) + with job: + Join()(double.pod(_dict_source(), label="dbl"), _arrow_source(), label="joined") + increment.pod( + CSVSource(str(csv), tag_columns=["id"], source_id="csrc"), label="inc" + ) + if cache_mode is not None: + job.nodes["joined"].attach_databases( + pipeline_database=db.at(JOB), cache_mode=cache_mode + ) + job.run() + path = tmp_path / "job.json" + job.save(path) + return path + + +def _snapshot(db: Any) -> dict[tuple[str, ...], int]: + db.flush() + counts: dict[tuple[str, ...], int] = {} + for path in db.list_table_paths(): + records = db.get_all_records(path) + counts[path] = 0 if records is None else records.num_rows + return counts + + +def _bound(path: Path, db: Any) -> PipelineJob: + loaded = PipelineJob.load(path, database=db) + loaded.bind(sources={"dsrc": _dict_source(), "asrc": _arrow_source()}) + return loaded + + +@pytest.mark.parametrize("orchestrator", ["sync", "async"]) +def test_a_load_only_run_fills_a_loaded_job_and_writes_nothing( + tmp_path: Path, orchestrator: str +) -> None: + db = InMemoryArrowDatabase() + path = _save(tmp_path, db) + before = _snapshot(db) + loaded = _bound(path, db) + for label in ("dbl", "joined", "inc"): + assert loaded.nodes[label].as_table().num_rows == 0, label + if orchestrator == "sync": + summary = loaded.run(load_only=True) + else: + summary = asyncio.run(loaded.run_async(load_only=True)) + assert summary.load_only + assert summary.nodes["dbl"].reused == 2 + assert summary.nodes["inc"].reused == 2 + assert _snapshot(db) == before + dbl = {r["id"]: r["dbl"] for r in loaded.nodes["dbl"].as_table().to_pylist()} + assert dbl == {0: 10, 1: 20} + joined = loaded.nodes["joined"].as_table().to_pylist() + assert sorted((r["id"], r["dbl"], r["y"]) for r in joined) == [ + (0, 10, 3), + (1, 20, 4), + ] + assert loaded.nodes["dbl"].head().status is ViewStatus.LOADED_ONLY + + +def test_nodes_below_an_unbound_source_stay_empty(tmp_path: Path) -> None: + db = InMemoryArrowDatabase() + loaded = PipelineJob.load(_save(tmp_path, db), database=db) + summary = loaded.run(load_only=True) + # The CSV source is rebuilt from its file; the in-memory ones are not saved. + assert loaded.nodes["inc"].as_table().num_rows == 2 + assert loaded.nodes["dbl"].as_table().num_rows == 0 + assert loaded.nodes["joined"].as_table().num_rows == 0 + assert "dbl" not in summary.nodes + + +def test_a_loaded_job_still_cannot_compute(tmp_path: Path) -> None: + db = InMemoryArrowDatabase() + loaded = _bound(_save(tmp_path, db), db) + with pytest.raises(RuntimeError, match=r"run\(load_only=True\)"): + loaded.run() + with pytest.raises(RuntimeError, match=r"run\(load_only=True\)"): + loaded.nodes["dbl"].run() + + +def test_a_loaded_node_runs_load_only_with_what_it_depends_on(tmp_path: Path) -> None: + db = InMemoryArrowDatabase() + loaded = _bound(_save(tmp_path, db), db) + view = loaded.nodes["joined"].run(load_only=True) + assert view.as_table().num_rows == 2 + assert loaded.nodes["dbl"].as_table().num_rows == 2 + summary = loaded.nodes["joined"].last_run + assert summary is not None and summary.load_only + + +def test_a_loaded_replay_operator_replays_its_run_in_a_load_only_run( + tmp_path: Path, +) -> None: + db = InMemoryArrowDatabase() + path = _save(tmp_path, db, cache_mode=CacheMode.LOG) + loaded = _bound(path, db) + loaded.nodes["joined"].attach_databases( + pipeline_database=db.at(JOB), cache_mode=CacheMode.REPLAY + ) + before = _snapshot(db) + loaded.run(load_only=True) + assert _snapshot(db) == before + assert loaded.nodes["joined"].as_table().num_rows == 2 From ee3e485eb55ede907a3540b3d20564afbca4264d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 22:16:08 -0800 Subject: [PATCH 24/48] test(objective): a second node runs before it is read (ITL-1095) Co-Authored-By: Claude Opus 5.5 --- superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md | 2 +- test-objective/integration/test_caching_flows.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md b/superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md index 6d8904834..24aae79a1 100644 --- a/superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md +++ b/superpowers/plans/2026-11-13-itl-1095-node-current-run-view.md @@ -1,6 +1,6 @@ # ITL-1095: a job node shows the current run; load-only run -Status: decisions received (see section 2). Implementing in four stages, one commit (or more) each, pushed to the draft PR. +Status: all four stages implemented on draft PR #380 (stage 1 da66b836, stage 2 c90eb966, stage 3 b693b50e, stage 4 3610fd20). Next: wait for #378 to merge, merge main, CI, CodeRabbit. Base: PR #378 branch head `2e9972e0` (ITL-1093). Branch: `eywalker/itl-1095-design-what-a-job-node-shows-the-current`. Rule (developer, November 13, 2026): a job node itself (`as_table`, `iter_data`, dataframe diff --git a/test-objective/integration/test_caching_flows.py b/test-objective/integration/test_caching_flows.py index 6cdca8a8f..ed91d25e4 100644 --- a/test-objective/integration/test_caching_flows.py +++ b/test-objective/integration/test_caching_flows.py @@ -85,6 +85,8 @@ def test_second_run_uses_cache(self): pipeline_database=pipeline_db, result_database=result_db, ) + # A node shows rows only after a run (ITL-1095); this run reads them all. + node2.run() data = list(node2.iter_data()) assert len(data) == 3 From 40c54f0950ceaec176ecd284ce5143cc11ab7841 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 22:26:46 -0800 Subject: [PATCH 25/48] test(arrays): read the v0.2-store jobs after a load-only run (ITL-1095) Co-Authored-By: Claude Opus 5.5 --- tests/test_arrays/test_array_v02_store.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_arrays/test_array_v02_store.py b/tests/test_arrays/test_array_v02_store.py index da43e93be..8706caa41 100644 --- a/tests/test_arrays/test_array_v02_store.py +++ b/tests/test_arrays/test_array_v02_store.py @@ -103,6 +103,10 @@ def test_v02_rows_are_ignored_recomputed_and_kept( # Read through a newly built job: a Delta database object that read a table before # the run does not see the rows the run added to it (not specific to hash labels). stored = build_job(store) + # A node shows rows only after a run (ITL-1095): a load-only run reads them from the + # store and computes nothing. + stored.run(orchestrator=orchestrator, load_only=True) + assert CALLS == ["plus_two"] * len(ARRAYS) rows = [ (t.as_dict()["k"], d.as_dict()["y"]) for t, d in stored.nodes["double"].iter_data() @@ -154,5 +158,8 @@ def test_second_run_reuses_the_rows_of_this_version(store: Path) -> None: assert CALLS == [] stored = build_job(store) + # A node shows rows only after a run (ITL-1095): a load-only run reads them. + stored.run(load_only=True) + assert CALLS == [] for label in ("double", "total", "direct_sum", "plus_one", "after_drop"): assert stored.nodes[label].as_table().num_rows == len(ARRAYS), label From 74a2ddf5f4596a883f09a99dcc0795eabb237f50 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 22:28:40 -0800 Subject: [PATCH 26/48] fix(pipeline): type-narrowing in the node.run() cascade (ITL-1095) Co-Authored-By: Claude Opus 5.5 --- src/orcapod/pipeline/node_run.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/orcapod/pipeline/node_run.py b/src/orcapod/pipeline/node_run.py index bfeae0e43..a7d4b1fd9 100644 --- a/src/orcapod/pipeline/node_run.py +++ b/src/orcapod/pipeline/node_run.py @@ -114,20 +114,26 @@ def run_with_dependencies(target: Any, *, load_only: bool) -> RunSummary: held: dict[Any, list[Any]] = {} seen: list[Any] = [] + def is_current(node: Any) -> bool: + """Whether ``node`` holds a run this run does not redo.""" + return isinstance(node, RunHoldingNode) and node.holds_current_run( + load_only=load_only + ) + + def is_source(node: Any) -> bool: + """Whether ``node`` is a job's source slot: read by the run, never held.""" + return isinstance(node, SourceJobNode) + def visit(node: Any) -> None: """Add ``node`` and, unless it is held or a source, what it depends on.""" if any(node is other for other in seen): return seen.append(node) dag.add_node(node) - if ( - node is not target - and isinstance(node, RunHoldingNode) - and node.holds_current_run(load_only=load_only) - ): + if node is not target and is_current(node): held[node] = list(node.iter_data()) return - if isinstance(node, SourceJobNode): + if is_source(node): return if not _is_job_node(node): # A stream outside the job: its rows, read as the node's run reads them. From 32bb76639b5126e3d8d510084cc96e90eca647b9 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 22:34:39 -0800 Subject: [PATCH 27/48] fix(function_node): an async run whose output close fails defines no rows (ITL-1093) Addresses the third CodeRabbit review of PR #378 (outside-diff finding). Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/function_node.py | 7 ++- .../test_job_node_rows_after_run.py | 63 +++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 1f81a2f34..54f83f8b4 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -3694,4 +3694,9 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: self.clear_cache() raise finally: - await output.close() + try: + await output.close() + except BaseException: + # A close that raises or is cancelled fails the run: no rows. + self.clear_cache() + raise diff --git a/tests/test_pipeline/test_job_node_rows_after_run.py b/tests/test_pipeline/test_job_node_rows_after_run.py index e45ff97bc..c31e7c487 100644 --- a/tests/test_pipeline/test_job_node_rows_after_run.py +++ b/tests/test_pipeline/test_job_node_rows_after_run.py @@ -475,3 +475,66 @@ def on_node_end(self, node_label: str, node_hash: str) -> None: job.run(orchestrator=_orchestrator(kind), observer=FailsAtNodeEnd()) # Every stored row (the 6-row run's), read from the store. assert _values(node, lambda r: r["k"]) == [0, 1, 2, 3, 4, 5] + + +class TestAsyncOutputCloseFails: + """An async run whose output cannot be closed defines no rows (ITL-1093).""" + + def test_failed_close_leaves_no_run_rows(self) -> None: + """A close that raises after every row was sent leaves the node with no run.""" + import asyncio + + import pyarrow as pa + + from orcapod.channels import Channel + from orcapod.core.nodes import FunctionJobNode + from orcapod.core.streams import ArrowTableStream + + class CloseFails: + """An output writer that keeps what it is sent and raises on close.""" + + def __init__(self) -> None: + """Start with no rows.""" + self.sent: list[Any] = [] + + async def send(self, item: Any) -> None: + """Keep ``item``.""" + self.sent.append(item) + + async def close(self) -> None: + """Raise, as a closed or broken channel would.""" + raise OSError("output cannot be closed") + + schema = pa.schema( + [ + pa.field("k", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] + ) + table = pa.Table.from_arrays( + [pa.array([0, 1], pa.int64()), pa.array([5, 6], pa.int64())], + schema=schema, + ) + node = FunctionJobNode( + inc.pod, # pyright: ignore[reportArgumentType] # the decorator's pod type is not declared as FunctionPodProtocol + ArrowTableStream(table, tag_columns=["k"]), + pipeline_database=InMemoryArrowDatabase(), + result_database=InMemoryArrowDatabase(), + ) + output = CloseFails() + + async def run() -> None: + """Run the node over its upstream's rows into ``output``.""" + inputs = Channel(buffer_size=16) + # The input stream is the node's own upstream (a test of async_execute). + for item in node.upstreams[0].iter_data(): + await inputs.writer.send(item) + await inputs.writer.close() + await node.async_execute(inputs.reader, output) # pyright: ignore[reportArgumentType] # a test double of the writer + + with pytest.raises(OSError, match="cannot be closed"): + asyncio.run(run()) + assert len(output.sent) == 2 + # No run is held: the rows are read from the store. + assert node.head(10).origin == "stored" + assert _values(node, lambda r: (r["k"], r["y"])) == [(0, 6), (1, 7)] From 2846dde605236c8845d82220956b388167ad61ac Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 01:30:58 -0800 Subject: [PATCH 28/48] feat(side_effects): a side-effect job node follows the job node state rule (ITL-1095) After #377 (ITL-1091) a SideEffectJobNode holds the rows its last run passed on; it now mixes in RunHoldingNode: stale before a run, transitive staleness, a current node delivers nothing on run() (clear_cache() to deliver again), run() cascades as every job node's does (shared run_job_node), and a load-only run passes its rows on, holds them as load-only and delivers nothing. The orchestrators no longer special-case side-effect nodes in a load-only run. Co-Authored-By: Claude Opus 5.5 --- .zed/rules | 7 +- CLAUDE.md | 7 +- changelog.d/ITL-1095.md | 1 + src/orcapod/core/nodes/function_node.py | 57 +++++++--- src/orcapod/core/nodes/staleness.py | 5 +- src/orcapod/pipeline/async_orchestrator.py | 39 +++---- src/orcapod/pipeline/sync_orchestrator.py | 26 ++--- src/orcapod/protocols/node_protocols.py | 2 + src/orcapod/side_effects.py | 103 +++++++++++++++--- .../test_side_effect_job_node_reads.py | 13 ++- tests/test_core/streams/test_run.py | 7 +- tests/test_pipeline/test_node_run_cascade.py | 28 +++++ 12 files changed, 213 insertions(+), 82 deletions(-) diff --git a/.zed/rules b/.zed/rules index a9d2c6d4f..dd15f3ebd 100644 --- a/.zed/rules +++ b/.zed/rules @@ -804,9 +804,10 @@ data. The rules: A loaded job (PipelineJob.load) runs only with load_only=True. - Empty data → ArrowTableSource raises ValueError("Table is empty"). - A side-effect job node delivers only when it runs (developer, November 13, 2026, ITL-1091): - job.run(), node.run() (the job run's work for that node: with track_completion only - what the invocation log does not record, without it every row again), execute, - async_execute. Every read (iter_data, as_table, dataframes, flow, head, preview, a + job.run(), node.run() (state-driven, ITL-1095: a node holding a current run delivers + nothing, so clear_cache() it first to deliver again; otherwise the job run's work for that + node: with track_completion only what the invocation log does not record, without it every + row), execute, async_execute; a load-only run delivers nothing. Every read (iter_data, as_table, dataframes, flow, head, preview, a downstream node's read) returns the rows its last run in this process passed on, held in memory (SideEffectJobNode._held_rows), and delivers nothing; with none (no run, a run that raised, or stale by is_stale) a read returns zero rows. Never route a job node's read to diff --git a/CLAUDE.md b/CLAUDE.md index 54b615442..1ada669b9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -846,9 +846,10 @@ data. The rules: A loaded job (`PipelineJob.load`) runs only with `load_only=True`. - Empty data → `ArrowTableSource` raises `ValueError("Table is empty")`. - A side-effect job node delivers only when it runs (developer, November 13, 2026, ITL-1091): - `job.run()`, `node.run()` (the job run's work for that node: with `track_completion` only - what the invocation log does not record, without it every row again), `execute`, - `async_execute`. Every read (`iter_data`, `as_table`, dataframes, `flow`, `head`, preview, a + `job.run()`, `node.run()` (state-driven, ITL-1095: a node holding a current run delivers + nothing, so `clear_cache()` it first to deliver again; otherwise the job run's work for that + node: with `track_completion` only what the invocation log does not record, without it every + row), `execute`, `async_execute`; a load-only run delivers nothing. Every read (`iter_data`, `as_table`, dataframes, `flow`, `head`, preview, a downstream node's read) returns the rows its last run in this process passed on, held in memory (`SideEffectJobNode._held_rows`), and delivers nothing; with none (no run, a run that raised, or stale by `is_stale`) a read returns zero rows. Never route a job node's read to diff --git a/changelog.d/ITL-1095.md b/changelog.d/ITL-1095.md index 543769b5f..7675b59a9 100644 --- a/changelog.d/ITL-1095.md +++ b/changelog.d/ITL-1095.md @@ -4,6 +4,7 @@ - A function or operator job node that has not run in this process is stale (`is_stale` is `True`; it has no modified time), and a job node is stale also when any function or operator job node it depends on is. Binding a source to a job's source slot (`job.bind(sources=...)`) marks the slot modified, so every node fed by it is stale and empty until it runs again. - A loaded operator node no longer raises from `as_table()` or `iter_data()`: it is empty, like every node that has not run. Its stored view (`as_source()`, `get_all_records()`) raises instead when the job file does not record its run's content hash and the store holds nothing under the hash the loaded job computes (ITL-1043). - A job node's `run()` is state-driven and runs what the node depends on (developer, November 13, 2026; ITL-1095). A node that holds a completed run and is not stale runs nothing: `run()` returns the rows it holds. Otherwise every job node it depends on that holds no current run runs first, each once, as `job.run()` would run that part of the job, then the node; the run's writes are committed when it ends (before, `node.run()` read an upstream node's stored rows, or nothing, and left its last pipeline record and a `LOG` operator's output uncommitted). `run()` of a node whose function pod is a placeholder, or that was loaded without one, raises unless `load_only=True`; a `CACHE_ONLY` node's `run()` raises (its input is unavailable): read its stored results with `as_source()`. A job node's `run()` takes one keyword-only argument, `load_only`; every other stream's `run()` still takes none. +- A side-effect job node follows the same rule: it is stale before a run, and while it holds a current run its `run()` (or a downstream node's `run()`) delivers nothing; to deliver again without `track_completion`, call `clear_cache()` on it first (`job.run()` still delivers as before). A load-only run passes its rows on and delivers nothing. - If the commit at the end of `job.run()` or `node.run()` fails, every function and operator node of the run is cleared (not run, stale) before the error propagates. ### Added diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index d2758686b..8a3e93ed3 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -274,6 +274,44 @@ async def close(self) -> None: # --------------------------------------------------------------------------- +def run_job_node( + node: Any, + *, + load_only: bool, + run_alone: Callable[[], Any], + caller: Any, +) -> None: + """Run a job node as its ``run()`` does (developer, November 13, 2026; ITL-1095). + + Nothing runs when the node holds a current run (``holds_current_run``). Otherwise + a node that depends on job nodes runs with them (``run_with_dependencies``) and one + built over streams outside a job runs alone (``run_alone``). The run's summary, if + it gives one, is kept as the node's ``last_run``, and failed rows warn at the + caller's line, as ``job.run()`` warns. + + Args: + node: The job node (a ``RunHoldingNode``). + load_only: Whether to load stored results only. + run_alone: Runs the node over its input when no job node feeds it; returns its + ``RunSummary`` or ``None``. + caller: The frame of the ``run()`` caller. + """ + from orcapod.pipeline.node_run import depends_on_job_nodes, run_with_dependencies + from orcapod.pipeline.run_summary import warn_row_failures + + if node.holds_current_run(load_only=load_only): + return + if depends_on_job_nodes(node): + summary = run_with_dependencies(node, load_only=load_only) + else: + summary = run_alone() + if summary is None: + return + node._last_run = summary + if not summary.ok: + warn_row_failures(summary, caller) + + class FunctionNodeBase(StreamBase): """Shared base for ``FunctionNode`` and ``FunctionJobNode``. @@ -3447,11 +3485,6 @@ def run(self, *, load_only: bool = False) -> StreamView: ``load_only`` is false; or if its input is unavailable (``CACHE_ONLY``): read its stored rows with ``as_source()``. """ - from orcapod.pipeline.node_run import ( - depends_on_job_nodes, - run_with_dependencies, - ) - from orcapod.pipeline.run_summary import warn_row_failures from orcapod.pipeline.serialization import LoadStatus status = self._load_status @@ -3466,14 +3499,12 @@ def run(self, *, load_only: bool = False) -> StreamView: self._function_pod is None or status == LoadStatus.READ_ONLY ): raise self._cannot_compute_error() - if not self.holds_current_run(load_only=load_only): - if depends_on_job_nodes(self): - summary = run_with_dependencies(self, load_only=load_only) - else: - summary = self._run_alone(load_only=load_only) - self._last_run = summary - if not summary.ok: - warn_row_failures(summary, sys._getframe(1)) # pyright: ignore[reportPrivateUsage] # the caller's line is where the warning is shown + run_job_node( + self, + load_only=load_only, + run_alone=lambda: self._run_alone(load_only=load_only), + caller=sys._getframe(1), # pyright: ignore[reportPrivateUsage] # the caller's line is where a failure warning is shown + ) return self._run_view() def _run_alone(self, *, load_only: bool) -> Any: diff --git a/src/orcapod/core/nodes/staleness.py b/src/orcapod/core/nodes/staleness.py index 841a96fd0..fc461a009 100644 --- a/src/orcapod/core/nodes/staleness.py +++ b/src/orcapod/core/nodes/staleness.py @@ -15,9 +15,8 @@ class RunHoldingNode: """Mixin of the job nodes that hold the rows of their current run. - ``FunctionJobNode`` and ``OperatorJobNode`` mix it in ahead of their stream base, so - their ``is_stale`` covers their whole ancestry. Side-effect job nodes do not yet: - their own ``is_stale`` is redefined by ITL-1091. + ``FunctionJobNode``, ``OperatorJobNode`` and ``SideEffectJobNode`` mix it in ahead of + their stream base, so their ``is_stale`` covers their whole ancestry. """ # The summary of the node's last ``run()`` (with the nodes it ran first); a class diff --git a/src/orcapod/pipeline/async_orchestrator.py b/src/orcapod/pipeline/async_orchestrator.py index 592d6fdb7..6cc442198 100644 --- a/src/orcapod/pipeline/async_orchestrator.py +++ b/src/orcapod/pipeline/async_orchestrator.py @@ -31,7 +31,6 @@ ) if TYPE_CHECKING: - from orcapod.channels import WritableChannel from orcapod.pipeline.dag import GraphProtocol from orcapod.protocols.core_protocols import DataProtocol, TagProtocol from orcapod.protocols.observability_protocols import ExecutionObserverProtocol @@ -301,20 +300,19 @@ async def _run_async( "SideEffectNode expects exactly 1 upstream, got 0" ) input_reader = edge_readers[(predecessors[0], node)] - if load_only: - # Nothing is delivered in a load-only run: rows pass on. - _start_node(tg, node, _pass_through(input_reader, writer)) - else: - _start_node( - tg, - node, - node.async_execute( - [input_reader], - writer, - observer=effective_observer, - run_id=run_id, - ), - ) + # In a load-only run the node passes its rows on and delivers + # nothing. + _start_node( + tg, + node, + node.async_execute( + [input_reader], + writer, + observer=effective_observer, + run_id=run_id, + load_only=load_only, + ), + ) else: raise TypeError( f"Unknown node type: {getattr(node, 'node_type', None)!r}" @@ -610,17 +608,6 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: return NoOpLogger() -async def _pass_through( - reader: Any, writer: WritableChannel[tuple[TagProtocol, DataProtocol]] -) -> None: - """Forward every row of ``reader`` to ``writer``, then close ``writer``.""" - try: - async for item in reader: - await writer.send(item) - finally: - await writer.close() - - def _start_node( tg: asyncio.TaskGroup, node: Any, work: Coroutine[Any, Any, None] ) -> None: diff --git a/src/orcapod/pipeline/sync_orchestrator.py b/src/orcapod/pipeline/sync_orchestrator.py index bcca4f106..a02723c57 100644 --- a/src/orcapod/pipeline/sync_orchestrator.py +++ b/src/orcapod/pipeline/sync_orchestrator.py @@ -75,7 +75,7 @@ def run( load_only: Fill the nodes from what is stored and compute nothing (ITL-1095): function nodes load their inputs' stored results and skip the others, operators run without storing, and a side-effect node - passes its rows on without being called (nothing is delivered). + passes its rows on and delivers nothing. held_rows: Nodes of ``graph`` that are not run: each one's rows (the rows of its current run, or of a stream outside the job) feed its successors as if it had just run. Used by ``node.run()``, which runs @@ -128,19 +128,17 @@ def run( ) elif is_side_effect_node(node): upstream_buf = self._gather_upstream(node, graph, buffers) - if load_only: - # Nothing is delivered in a load-only run: the rows pass on. - buffers[node] = list(upstream_buf) - else: - upstream_node = list(graph.predecessors(node))[0] - input_stream = self._materialize_as_stream( - upstream_buf, upstream_node - ) - buffers[node] = node.execute( - input_stream, - observer=effective_observer, - run_id=run_id, - ) + upstream_node = list(graph.predecessors(node))[0] + input_stream = self._materialize_as_stream( + upstream_buf, upstream_node + ) + # In a load-only run the node passes its rows on and delivers nothing. + buffers[node] = node.execute( + input_stream, + observer=effective_observer, + run_id=run_id, + load_only=load_only, + ) else: raise TypeError( f"Unknown node type: {getattr(node, 'node_type', None)!r}" diff --git a/src/orcapod/protocols/node_protocols.py b/src/orcapod/protocols/node_protocols.py index c33a71c84..183d3a4ae 100644 --- a/src/orcapod/protocols/node_protocols.py +++ b/src/orcapod/protocols/node_protocols.py @@ -160,6 +160,7 @@ def execute( *, observer: "ExecutionObserverProtocol | None" = None, run_id: str | None = None, + load_only: bool = False, ) -> "list[tuple[TagProtocol, DataProtocol]]": ... async def async_execute( @@ -169,6 +170,7 @@ async def async_execute( *, observer: "ExecutionObserverProtocol | None" = None, run_id: str | None = None, + load_only: bool = False, ) -> None: ... def attach_databases( diff --git a/src/orcapod/side_effects.py b/src/orcapod/side_effects.py index fb9ad3d2e..6da628ced 100644 --- a/src/orcapod/side_effects.py +++ b/src/orcapod/side_effects.py @@ -18,7 +18,8 @@ from typing_extensions import override from orcapod.core.base import TraceableBase -from orcapod.core.nodes.function_node import _build_record_id_preimage +from orcapod.core.nodes.staleness import RunHoldingNode +from orcapod.core.nodes.function_node import _build_record_id_preimage, run_job_node from orcapod.core.streams.base import ( StreamBase, check_head_size, @@ -915,7 +916,7 @@ def node_uri(self) -> tuple[str, ...]: # --------------------------------------------------------------------------- -class SideEffectJobNode(SideEffectNode): +class SideEffectJobNode(RunHoldingNode, SideEffectNode): """DB-backed execution node for side-effect pods. Created at pipeline compile time by ``PipelineJob``. Receives a @@ -950,6 +951,28 @@ def __init__( # The rows the last completed run passed on, in the order it passed them; # ``None`` when no run in this process completed (ITL-1091). self._held_rows: list[tuple[TagProtocol, DataProtocol]] | None = None + # Whether those rows came from a load-only run, which delivers nothing. + self._run_load_only = False + # A node that has not run in this process holds nothing and is stale (ITL-1095). + self._set_modified_time(None) + + def _holds_run(self, *, load_only: bool) -> bool: + """Whether the node holds a run's rows; a load-only run's only for ``load_only``. + + A load-only run delivers nothing, so a full run does not take its rows as the + result of a delivery. + """ + return self._held_rows is not None and (load_only or not self._run_load_only) + + def clear_cache(self) -> None: + """Drop the rows the last run passed on: the node is stale and reads empty. + + Its next ``run()`` (or job run) delivers again what its invocation log does not + record (every row without ``track_completion``) (ITL-1095). + """ + self._held_rows = None + self._run_load_only = False + self._set_modified_time(None) def _refuse_log_of_another_data_context(self) -> None: """Refuse an invocation log holding deliveries of another data context. @@ -1036,6 +1059,7 @@ def execute( *, observer: ExecutionObserverProtocol | None = None, run_id: str | None = None, + load_only: bool = False, ) -> list[tuple[TagProtocol, DataProtocol]]: """Execute side-effect delivery for all rows in ``input_stream``. @@ -1043,6 +1067,8 @@ def execute( input_stream: Stream of ``(tag, data)`` pairs to process. observer: Optional execution observer (currently unused). run_id: Pipeline run identifier forwarded from the orchestrator. + load_only: A load-only run (ITL-1095): every row passes on and nothing is + delivered or logged. The node keeps the rows it passed on, which every later read returns without delivering (ITL-1091). A run that raises leaves the node with no rows. @@ -1055,6 +1081,10 @@ def execute( logged under another data context; nothing is delivered then. """ self._held_rows = None + if load_only: + rows = list(input_stream.iter_data()) + self._hold(rows, load_only=True) + return rows self._refuse_log_of_another_data_context() results: list[tuple[TagProtocol, DataProtocol]] = [] for tag, data in input_stream.iter_data(): @@ -1077,13 +1107,20 @@ def execute( self._hold(results) return results - def _hold(self, rows: list[tuple[TagProtocol, DataProtocol]]) -> None: + def _hold( + self, rows: list[tuple[TagProtocol, DataProtocol]], *, load_only: bool = False + ) -> None: """Keep the rows a completed run passed on and stamp the node as modified. Stamped after the upstream nodes, which stamp at the end of their own run, so the rows are current until an upstream changes again (``is_stale``). + + Args: + rows: The rows the run passed on. + load_only: Whether the run was load-only (it delivered nothing). """ self._held_rows = rows + self._run_load_only = load_only self._update_modified_time() def _current_rows(self) -> list[tuple[TagProtocol, DataProtocol]] | None: @@ -1150,18 +1187,25 @@ def head( ) @override - def run(self) -> StreamView: + def run(self, *, load_only: bool = False) -> StreamView: """Run the node as ``job.run()`` runs it and return a ``StreamView`` of its rows. - Reads the input and delivers through the invocation log, as a job run does: - with ``track_completion`` only the rows the log does not record as delivered are - delivered; without it every row is delivered again, as a second ``job.run()`` - delivers it. With no database attached nothing is logged and every row is - delivered. The node then holds the rows it passed on, which ``as_table()`` and - every other read return without delivering. The invocation log is committed - before ``run()`` returns, as at the end of a job run; after a failed delivery - the log rows of the deliveries that succeeded are committed too, and the - delivery's error is raised. + State-driven, as every job node's ``run()`` (developer, November 13, 2026; + ITL-1095): a node that holds the rows of a completed, non-stale run delivers + nothing and returns them; call ``clear_cache()`` first to deliver again. + Otherwise the job nodes it depends on that hold no current run run first, and + the node reads its input and delivers through the invocation log, as a job run + does: with ``track_completion`` only the rows the log does not record as + delivered; without it every row. With no database attached nothing is logged + and every row is delivered. The node then holds the rows it passed on, which + ``as_table()`` and every other read return without delivering. The run's writes + (the invocation log included) are committed before ``run()`` returns; after a + failed delivery the log rows of the deliveries that succeeded are committed too, + and the delivery's error is raised. + + Args: + load_only: Deliver nothing: the rows pass on, and the nodes run first load + their stored results only (as ``job.run(load_only=True)``). Returns: A ``StreamView`` of the rows this run passed on. @@ -1170,9 +1214,21 @@ def run(self) -> StreamView: SideEffectLogDataContextError: As for ``execute``. StoreWriteError: If committing the invocation log fails. """ + import sys + + run_job_node( + self, + load_only=load_only, + run_alone=lambda: self._run_alone(load_only=load_only), + caller=sys._getframe(1), # pyright: ignore[reportPrivateUsage] # the caller's line is where a failure warning is shown + ) + return self._view_of_rows(self.as_table(), "held") + + def _run_alone(self, *, load_only: bool) -> None: + """Run this node over an input stream that is not a job node; commit its log.""" database = self._pipeline_database try: - _ = self.execute(self._input_stream) + _ = self.execute(self._input_stream, load_only=load_only) except Exception as run_error: if database is not None: flush_best_effort(run_error, database) @@ -1184,12 +1240,11 @@ def run(self) -> StreamView: except StoreReadError: # A stored table could not be read: report the read failure itself. # The run raised, so it leaves the node no rows. - self._held_rows = None + self.clear_cache() raise except Exception as exc: - self._held_rows = None + self.clear_cache() raise StoreWriteError(database.base_path, flush=True) from exc - return self._view_of_rows(self.as_table(), "held") # ------------------------------------------------------------------ # Async execution @@ -1202,6 +1257,7 @@ async def async_execute( *, observer: ExecutionObserverProtocol | None = None, run_id: str | None = None, + load_only: bool = False, ) -> None: """Async side-effect delivery with semaphore-bounded concurrency. @@ -1222,6 +1278,8 @@ async def async_execute( output: Writable channel for pass-through ``(tag, data)`` pairs. observer: Optional execution observer (currently unused). run_id: Pipeline run identifier from the orchestrator. + load_only: A load-only run (ITL-1095): every row passes on and nothing is + delivered or logged. Raises: ValueError: If ``inputs`` does not contain exactly one channel. @@ -1237,6 +1295,17 @@ async def async_execute( ) self._held_rows = None + if load_only: + try: + rows: list[tuple[TagProtocol, DataProtocol]] = [] + async for item in inputs[0]: + rows.append(item) + await output.send(item) + # Held and stamped before the output closes (ITL-1042). + self._hold(rows, load_only=True) + finally: + await output.close() + return # This run's rows, in the order they were sent downstream. passed_on: list[tuple[TagProtocol, DataProtocol]] = [] # Makes recording a row and sending it one step: a send blocked by a full channel diff --git a/tests/test_core/side_effect_pod/test_side_effect_job_node_reads.py b/tests/test_core/side_effect_pod/test_side_effect_job_node_reads.py index 47e803a67..cb272bca9 100644 --- a/tests/test_core/side_effect_pod/test_side_effect_job_node_reads.py +++ b/tests/test_core/side_effect_pod/test_side_effect_job_node_reads.py @@ -184,13 +184,19 @@ def test_node_run_delivers_what_the_log_does_not_record( @pytest.mark.parametrize("orchestrator", ["sync", "async"]) -def test_node_run_without_track_completion_delivers_every_row_as_a_job_run_does( +def test_node_run_without_track_completion_delivers_again_only_after_clear_cache( orchestrator: Orchestrator, ) -> None: - """Without ``track_completion``, ``node.run()`` delivers every row again, as ``job.run()`` does.""" + """Without ``track_completion``, a node that holds a current run delivers nothing on + ``node.run()`` (ITL-1095); after ``clear_cache()`` it delivers every row again, as + ``job.run()`` does.""" job, sent = _job(track_completion=False) _run(job, orchestrator) view = job.nodes["delivered"].run() + assert len(sent) == ROWS + assert len(view) == ROWS + job.nodes["delivered"].clear_cache() + view = job.nodes["delivered"].run() assert len(sent) == 2 * ROWS assert len(view) == ROWS _run(job, orchestrator) @@ -319,6 +325,9 @@ def test_rows_held_from_a_run_are_dropped_when_an_upstream_changes( _run(job, orchestrator) node = job.nodes["delivered"] assert len(node.as_table()) == ROWS + # The upstream runs again: a node that holds a current run runs nothing (ITL-1095), + # so it is cleared first. + job.nodes["up"].clear_cache() _ = job.nodes["up"].run() for name, read in _reads(node).items(): _ = read() diff --git a/tests/test_core/streams/test_run.py b/tests/test_core/streams/test_run.py index 5df3b027a..c11660fcf 100644 --- a/tests/test_core/streams/test_run.py +++ b/tests/test_core/streams/test_run.py @@ -117,7 +117,12 @@ def test_every_stream_class_has_run_without_positional_arguments(cls: type) -> N assert callable(run), f"{cls.__qualname__} has no run()" parameters = inspect.signature(run).parameters names = list(parameters) - if cls.__name__ in ("FunctionJobNode", "OperatorJobNode", "SourceJobNode"): + if cls.__name__ in ( + "FunctionJobNode", + "OperatorJobNode", + "SideEffectJobNode", + "SourceJobNode", + ): assert names == ["self", "load_only"], f"{cls.__qualname__}.run{names}" load_only = parameters["load_only"] assert load_only.kind is inspect.Parameter.KEYWORD_ONLY diff --git a/tests/test_pipeline/test_node_run_cascade.py b/tests/test_pipeline/test_node_run_cascade.py index 4d518933b..5fd93ffff 100644 --- a/tests/test_pipeline/test_node_run_cascade.py +++ b/tests/test_pipeline/test_node_run_cascade.py @@ -288,6 +288,34 @@ def test_a_load_only_run_that_found_everything_stays_loaded_only(self) -> None: class TestSideEffects: + def test_a_side_effect_node_follows_the_state_rule(self) -> None: + """Stale before a run; while it holds a current run nothing re-delivers it.""" + job = _build(ROWS, InMemoryArrowDatabase(), with_tap=True) + tap = job.nodes["tap"] + assert tap.is_stale + _ = job.nodes["summed"].run() + assert len(DELIVERED) == 6 + assert tap.holds_current_run() + _ = job.nodes["summed"].run() + _ = tap.run() + assert len(DELIVERED) == 6 + tap.clear_cache() + assert job.nodes["summed"].is_stale + _ = job.nodes["summed"].run() + assert len(DELIVERED) == 12 + + def test_a_load_only_run_of_a_side_effect_node_holds_its_rows(self) -> None: + db = InMemoryArrowDatabase() + _build(ROWS, db, with_tap=True).run() + DELIVERED.clear() + job = _build(ROWS, db, with_tap=True) + _ = job.nodes["summed"].run(load_only=True) + tap = job.nodes["tap"] + assert DELIVERED == [] + assert tap.as_table().num_rows == 6 + assert tap.holds_current_run(load_only=True) + assert not tap.holds_current_run() + def test_a_full_cascade_delivers_a_load_only_cascade_does_not(self) -> None: db = InMemoryArrowDatabase() job = _build(ROWS, db, with_tap=True) From a1e1af763a9da764c9ffe9f199c380106c372444 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 01:41:20 -0800 Subject: [PATCH 29/48] docs(plans): ITL-1094 reproduction and proposal (WIP) Co-Authored-By: Claude Opus 5.5 --- ...-11-14-itl-1094-loaded-side-effect-node.md | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 superpowers/plans/2026-11-14-itl-1094-loaded-side-effect-node.md diff --git a/superpowers/plans/2026-11-14-itl-1094-loaded-side-effect-node.md b/superpowers/plans/2026-11-14-itl-1094-loaded-side-effect-node.md new file mode 100644 index 000000000..8c40a659a --- /dev/null +++ b/superpowers/plans/2026-11-14-itl-1094-loaded-side-effect-node.md @@ -0,0 +1,91 @@ +# ITL-1094: a loaded job restores its side-effect nodes, read-only + +Branch `eywalker/itl-1094-a-loaded-job-has-no-side-effect-nodes`, from `origin/main` `a9b50350`. +Status: PROPOSAL sent to the coordinator; waiting for the reply before any code. + +## Reproduction (main a9b50350, public interface) + +Job `s -> delivered (SideEffectPod) -> down (function pod) -> tail (SideEffectPod)`, run on a +Delta store, saved, loaded. + +- The saved file records each side-effect node like every other node, under the key of its + blueprint content hash, with exactly these fields: `node_type: "side_effect"`, `label`, + `content_hash` (blueprint hash), `pipeline_hash` (equal to the job node's), `output_schema` + (`tag`, `data`, `nullable`), `node_uri` (`["side_effects", ]`), `data_context_key`, + `config` (the Orcapod config, `null` when default). The edges `s -> delivered` and + `delivered -> down` are saved. `run.node_job_content_hashes` and `run.expected_rows` hold + no entry for a side-effect node. The pod config (`on_error`, `track_completion`, + `drop_on_failure`, `max_concurrency`) is not saved anywhere. +- `Pipeline.load` has no `side_effect` branch: the node is skipped. `loaded.nodes` is + `['s', 'down']`; `loaded.nodes['delivered']` and `['tail']` are `KeyError`. +- `down` after the load: its `upstreams` is `(None,)`, and the job DAG has neither edge + `s -> delivered` nor `delivered -> down`. Its content hash and pipeline hash EQUAL the saved + ones (both come from the file: the run block's live hash and the descriptor), so it reads + the same table: 3 rows. No silent identity change. +- Re-saving the loaded job writes a file WITHOUT the side-effect nodes but WITH their edges + (3 dangling edges); loading that file gives `['s', 'down']` again. +- On the #380 branch (head 0b4c2316), `loaded.bind(sources=...)` then + `loaded.run(load_only=True)` raises `ValueError: FunctionNode expects exactly 1 upstream, + got 0` at `down`: the dropped node breaks a load-only run of anything below it. +- v0.2 (format 0.1.0) saved side-effect nodes with the same fields minus + `output_schema.nullable`, with `semantic_v0.1` hashes; v0.2's load dropped them too. No test + fixture holds a v0.2 job file with a side-effect node. + +## Proposal + +A read-only side-effect node, built from its descriptor, as the function and operator stubs are. + +1. `SideEffectNode.from_descriptor(descriptor, input_stream)` (blueprint, used by + `Pipeline.load`) and `SideEffectJobNode.from_descriptor(descriptor, input_stream)` (used by + `PipelineJob.load`): no pod (`_pod = None`); label, content hash, pipeline hash, output + schema (with the saved nullability, or unrecorded), node uri, data context and Orcapod + config from the descriptor; upstream = the restored upstream node, so the DAG edges + `s -> delivered -> down` are restored and `down.upstreams == (delivered,)`. +2. Identity: `pipeline_hash()` = the saved one (equal to the job node's; it names the + invocation log table `side_effects//schema:/tdb_v2`). `content_hash()` = the + descriptor's `content_hash` (the blueprint hash the file records). The live job node's + content hash (data-inclusive) is not in the file. Nothing stored is keyed by a side-effect + node's content hash (log path and record ids use the pipeline hash and the row), so no + table depends on it. Option B (not recommended for the tag): record it in + `run.node_job_content_hashes` at save (an existing label -> hash map; no new key; older + loaders ignore extra entries; older files fall back to the descriptor hash). It changes + what the file records, so it is the coordinator's call. +3. Never delivers: no pod function. `execute`, `async_execute` and `node.run()` raise a + `RuntimeError` saying the node was loaded and has no side-effect function. Reads follow + `SideEffectJobNode` (#377): no held rows, so zero rows; `head()` not computed. + `loaded.run()` raises as today (before any node runs). +4. Pod config and completion log: not restored; the file does not hold the pod config, and + restoring it would be a saved-format change (not proposed). The database is attached as for + any `SideEffectJobNode` (path known from node uri + pipeline hash), but the node never + reads or writes the log. No public reader is added. +5. Re-save of a loaded job writes the node's descriptor back unchanged (and the edges become + consistent again). +6. No new field in the saved file; no hash, golden value or stored table changes. + +## After #380 (expected to merge first; then merge main here) + +- #380's orchestrators pass a side-effect node's rows on in a load-only run without calling it + (sync: `buffers[node] = upstream_buf`; async: `_pass_through`), so the restored node needs + no code change: `loaded.run(load_only=True)` passes the rows through `delivered` to `down`, + delivers nothing, writes nothing. On #380 head 0b4c2316 the side-effect node itself is NOT + filled ("node reads empty", its D4); the brief's "holds them as loaded only" is not what + #380 does yet. +- At the merge: docs section "Saving and loading a job" conflicts (both rewrite it); tests + gain a load-only case (bind source, `loaded.run(load_only=True)`, `down` loads 3 rows, + zero deliveries, side-effect node empty). + +## Tasks (after approval) + +1. Failing tests, public interface: `tests/test_pipeline/loaded_job/test_loaded_side_effect_nodes.py` + - side-effect node in the middle and at the end: `loaded.nodes` lists them with label, + content hash (= saved descriptor), pipeline hash (= saved job node), upstream. + - `down`'s content/pipeline hash equal the saved job node's, `down.upstreams` is the + restored node. + - no read (`iter_data`, `as_table`, `head`, dataframes, `flow`) or run (`loaded.run()`, + `node.run()`) delivers: count deliveries; reads give zero rows; runs raise. + - re-save and reload keeps them; `Pipeline.load` lists them. + - a 0.1.0-shaped file (no nullable) restores them. +2. Fix: `from_descriptor` on both classes; `Pipeline.load` and `PipelineJob.load` branches. +3. Docs: `docs/concepts/function-pods.md` "Saving and loading a job"; changelog fragment + `changelog.d/ITL-1094.md` (Fixed); `design_issues/ITL-1094-.md`. +4. Targeted tests (loaded_job, side_effect_pod, serialization, golden), ruff, basedpyright. From 42817553223de286fc74d2e923c1ba1a470a0412 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 01:43:50 -0800 Subject: [PATCH 30/48] test(pipeline): failing tests: job.run() follows the state rule (ITL-1095) Co-Authored-By: Claude Opus 5.5 --- .../test_pipeline/test_job_run_state_rule.py | 210 ++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 tests/test_pipeline/test_job_run_state_rule.py diff --git a/tests/test_pipeline/test_job_run_state_rule.py b/tests/test_pipeline/test_job_run_state_rule.py new file mode 100644 index 000000000..32761f012 --- /dev/null +++ b/tests/test_pipeline/test_job_run_state_rule.py @@ -0,0 +1,210 @@ +"""``job.run()`` follows the state rule of ``node.run()`` (ITL-1095). + +Developer, November 14, 2026 ("let's align now"): ``job.run()`` and ``run_async()`` apply +the same rule as ``node.run()``. A node that holds a current completed run (not stale; for +a full run, not filled by a load-only run) is not run again: its rows feed the nodes after +it. Never-run, stale and (for a full run) load-only-filled nodes run. So a second +``job.run()`` of an unchanged job computes nothing, reads and writes nothing in the store +and delivers no side effect. +""" + +from __future__ import annotations + +import asyncio +from typing import Any +from unittest import mock + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.errors import StoreWriteError +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.side_effects import SideEffectPod, tap_pod +from orcapod.sources import DictSource + +CALLS: list[str] = [] +DELIVERED: list[int] = [] + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + """Return ``x + 1``.""" + CALLS.append("inc") + return x + 1 + + +@function_pod(output_keys="total") +def total(y: list[int]) -> int: + """Return the sum of a group's values.""" + CALLS.append("total") + return sum(y) + + +def _deliver(y: int, ctx: Any) -> None: + """A side effect: record the delivered value.""" + DELIVERED.append(y) + + +ROWS = [{"g": i % 2, "k": i, "x": 10 * i} for i in range(6)] +ORCHESTRATORS = {"sync": SyncPipelineOrchestrator, "async": AsyncPipelineOrchestrator} + + +@pytest.fixture(autouse=True) +def _reset() -> None: + CALLS.clear() + DELIVERED.clear() + + +def _source(data: list[dict[str, int]]) -> DictSource: + return DictSource(data, tag_columns=["g", "k"], source_id="s") + + +def _build(data: list[dict[str, int]], db: Any) -> PipelineJob: + """``rowwise`` -> ``tap`` (side effect, no completion tracking) -> ``grouped`` -> + ``summed``.""" + job = PipelineJob(name="j", database=db) + with job: + rows = inc.pod(_source(data), label="rowwise") + tap = tap_pod(_deliver) + assert isinstance(tap, SideEffectPod) + tapped: Any = tap.process(rows, label="tap") + grouped = tapped.group_by("g", label="grouped") + total.pod(grouped, label="summed") + return job + + +def _run(job: PipelineJob, orchestrator: str, **kwargs: Any) -> Any: + return job.run(orchestrator=ORCHESTRATORS[orchestrator](), **kwargs) + + +def _totals(job: PipelineJob) -> list[tuple[int, int]]: + return sorted( + (r["g"], r["total"]) for r in job.nodes["summed"].as_table().to_pylist() + ) + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_a_second_run_of_an_unchanged_job_does_nothing(orchestrator: str) -> None: + job = _build(ROWS, InMemoryArrowDatabase()) + _run(job, orchestrator) + assert len(DELIVERED) == 6 + CALLS.clear() + DELIVERED.clear() + with ( + mock.patch.object( + InMemoryArrowDatabase, + "get_all_records", + side_effect=AssertionError("store read"), + ), + mock.patch.object( + InMemoryArrowDatabase, + "add_records", + side_effect=AssertionError("store write"), + ), + mock.patch.object( + DictSource, "iter_data", side_effect=AssertionError("source read") + ), + ): + summary = _run(job, orchestrator) + assert CALLS == [] + assert DELIVERED == [] + assert _totals(job) == [(0, 63), (1, 93)] + rowwise = summary.nodes["rowwise"] + assert (rowwise.computed, rowwise.reused, rowwise.held) == (0, 0, 6) + assert summary.nodes["summed"].held == 2 + assert "rowwise: 6 rows: held from the previous run" in repr(summary) + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_a_cleared_node_runs_again_with_the_nodes_below_it(orchestrator: str) -> None: + """Clearing ``summed`` runs it alone, from the rows ``grouped`` holds.""" + job = _build(ROWS, InMemoryArrowDatabase()) + _run(job, orchestrator) + CALLS.clear() + DELIVERED.clear() + job.nodes["summed"].clear_cache() + summary = _run(job, orchestrator) + assert CALLS == [] # its results are stored: read, not computed + assert DELIVERED == [] + assert summary.nodes["summed"].reused == 2 + assert summary.nodes["rowwise"].held == 6 + assert _totals(job) == [(0, 63), (1, 93)] + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_clearing_a_side_effect_node_delivers_it_again(orchestrator: str) -> None: + job = _build(ROWS, InMemoryArrowDatabase()) + _run(job, orchestrator) + DELIVERED.clear() + job.nodes["tap"].clear_cache() + _run(job, orchestrator) + assert len(DELIVERED) == 6 + assert CALLS.count("inc") == 6 # only the first run computed + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_a_rebound_source_runs_every_node_below_it(orchestrator: str) -> None: + job = _build(ROWS[:3], InMemoryArrowDatabase()) + _run(job, orchestrator) + CALLS.clear() + job.bind(sources={next(iter(job.sources)): _source(ROWS)}) + _run(job, orchestrator) + assert CALLS.count("inc") == 3 + assert _totals(job) == [(0, 63), (1, 93)] + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_a_full_run_after_a_load_only_run_runs_the_load_only_nodes( + orchestrator: str, +) -> None: + db = InMemoryArrowDatabase() + _run(_build(ROWS[:3], db), orchestrator) + CALLS.clear() + DELIVERED.clear() + job = _build(ROWS, db) + _run(job, orchestrator, load_only=True) + assert CALLS == [] and DELIVERED == [] + _run(job, orchestrator) + assert CALLS.count("inc") == 3 + assert len(DELIVERED) == 6 + assert _totals(job) == [(0, 63), (1, 93)] + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_a_load_only_run_after_a_full_run_does_nothing(orchestrator: str) -> None: + job = _build(ROWS, InMemoryArrowDatabase()) + _run(job, orchestrator) + CALLS.clear() + summary = _run(job, orchestrator, load_only=True) + assert CALLS == [] + assert summary.nodes["rowwise"].held == 6 + + +def test_run_async_follows_the_rule() -> None: + job = _build(ROWS, InMemoryArrowDatabase()) + asyncio.run(job.run_async()) + CALLS.clear() + DELIVERED.clear() + summary = asyncio.run(job.run_async()) + assert CALLS == [] and DELIVERED == [] + assert summary.nodes["summed"].held == 2 + + +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_a_failed_commit_clears_only_the_nodes_that_ran(orchestrator: str) -> None: + job = _build(ROWS, InMemoryArrowDatabase()) + _run(job, orchestrator) + job.nodes["summed"].clear_cache() + + def fail(self: Any) -> None: + raise OSError("disk full") + + with ( + mock.patch.object(InMemoryArrowDatabase, "flush", fail), + pytest.raises(StoreWriteError), + ): + _run(job, orchestrator) + assert job.nodes["summed"].as_table().num_rows == 0 + assert job.nodes["rowwise"].as_table().num_rows == 6 + assert job.nodes["grouped"].as_table().num_rows == 2 From 62694bc1a4ab025e00adaed016e4739a0c541c7e Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 01:47:05 -0800 Subject: [PATCH 31/48] fix(pipeline): a loaded job restores its side-effect nodes, read-only (ITL-1094) (WIP) Co-Authored-By: Claude Opus 5.5 --- src/orcapod/pipeline/graph.py | 13 + src/orcapod/pipeline/job.py | 11 +- src/orcapod/side_effects.py | 349 +++++++++++++++++- .../test_loaded_side_effect_nodes.py | 234 ++++++++++++ 4 files changed, 604 insertions(+), 3 deletions(-) create mode 100644 tests/test_pipeline/loaded_job/test_loaded_side_effect_nodes.py diff --git a/src/orcapod/pipeline/graph.py b/src/orcapod/pipeline/graph.py index 3d82e45ec..a8795653e 100644 --- a/src/orcapod/pipeline/graph.py +++ b/src/orcapod/pipeline/graph.py @@ -466,6 +466,19 @@ def load(cls, path: str | Path) -> "Pipeline": ) reconstructed[node_hash] = node + elif node_type == "side_effect": + # Restored read-only: the file holds no side-effect function (ITL-1094). + up_hashes = upstream_map.get(node_hash, []) + upstream_node = reconstructed.get(up_hashes[0]) if up_hashes else None + if upstream_node is None: + raise ValueError( + f"The saved side-effect node {descriptor.get('label')!r} has " + "no upstream node in the file." + ) + reconstructed[node_hash] = SideEffectNode.from_descriptor( + descriptor, input_stream=upstream_node + ) + # Build Pipeline instance pipeline = cls(name=name, auto_compile=False) pipeline._unrecorded_output_nullability = unrecorded_output_nullability diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index c38146edc..45c4e58b8 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -15,7 +15,7 @@ from orcapod.core.nodes.function_node import FunctionJobNode from orcapod.core.nodes.operator_node import OperatorJobNode from orcapod.core.nodes.source_node import SourceJobNode -from orcapod.side_effects import SideEffectJobNode +from orcapod.side_effects import SideEffectJobNode, SideEffectNode from orcapod.core.tracker import AutoRegisteringContextBasedTracker from orcapod.contexts.stored import ( report_unclaimed_tables, @@ -1623,6 +1623,15 @@ def load( ) job._persistent_node_map[node_hash] = ojn + elif isinstance(bp_node, SideEffectNode): + # Read-only: it never delivers (ITL-1094). Its upstream is the restored + # job node, so the nodes below it keep their position in the graph. + preds = list(job._hash_graph.predecessors(node_hash)) + job._persistent_node_map[node_hash] = SideEffectJobNode.from_descriptor( + getattr(bp_node, "_descriptor", None) or {}, + input_stream=job._persistent_node_map[preds[0]], + ) + # Update _nodes to point to job nodes (FunctionJobNode/OperatorJobNode stubs) # rather than blueprint nodes, so that job.nodes["label"].get_all_records() # works on a loaded job. diff --git a/src/orcapod/side_effects.py b/src/orcapod/side_effects.py index fb9ad3d2e..180fee56f 100644 --- a/src/orcapod/side_effects.py +++ b/src/orcapod/side_effects.py @@ -66,6 +66,7 @@ ) from orcapod.protocols.database_protocols import ArrowDatabaseProtocol from orcapod.protocols.observability_protocols import ExecutionObserverProtocol + from orcapod.contexts import DataContext from orcapod.types import ContentHash, Schema else: pa = LazyModule("pyarrow") @@ -800,7 +801,8 @@ def __init__( # ------------------------------------------------------------------ @property - def producer(self) -> SideEffectPod: # type: ignore[override] + def producer(self) -> SideEffectPod | None: # type: ignore[override] + """The node's pod; ``None`` for a node loaded from a saved file (ITL-1094).""" return self._pod @property @@ -897,9 +899,13 @@ def as_table( column_config, tag_keys=lambda: self.keys()[0], declared_schema=lambda: self.output_schema(all_info=True), - type_converter=self._pod.data_context.type_converter, + type_converter=self._rows_data_context().type_converter, ) + def _rows_data_context(self) -> DataContext: + """The data context the node's rows are typed and hashed with: its pod's.""" + return self._pod.data_context + @property def node_uri(self) -> tuple[str, ...]: """Canonical URI tuple identifying this side-effect node. @@ -909,6 +915,31 @@ def node_uri(self) -> tuple[str, ...]: """ return self._pod.uri + @classmethod + def from_descriptor( + cls, descriptor: dict[str, Any], input_stream: StreamProtocol + ) -> SideEffectNode: + """Restore a side-effect node of a saved pipeline, read-only (ITL-1094). + + The saved file holds no side-effect function and no pod configuration, so the + node cannot deliver: every read or run that would deliver raises. Its label, + content hash, pipeline hash, output schema, node uri, data context and Orcapod + configuration are the saved ones. + + Args: + descriptor: The node's entry in the saved file (``node_type`` + ``"side_effect"``). + input_stream: The restored node upstream of it. + + Returns: + The read-only node. + + Raises: + ValueError: If the descriptor lacks ``content_hash``, ``pipeline_hash`` or + ``node_uri``. + """ + return _LoadedSideEffectNode(descriptor, input_stream) + # --------------------------------------------------------------------------- # SideEffectJobNode — DB-backed execution node @@ -997,6 +1028,30 @@ def _refuse_log_of_another_data_context(self) -> None: raise self._current_log_checked = True + @classmethod + @override + def from_descriptor( + cls, descriptor: dict[str, Any], input_stream: StreamProtocol + ) -> SideEffectJobNode: + """Restore a side-effect node of a saved job, read-only (ITL-1094). + + Used by ``PipelineJob.load``. The node has no side-effect function: it never + delivers and never writes its invocation log. Its reads follow every + ``SideEffectJobNode``: with no rows from a run in this process, none. + ``execute``, ``async_execute`` and ``run()`` raise. + + Args: + descriptor: The node's entry in the saved file. + input_stream: The restored job node upstream of it. + + Returns: + The read-only job node. + + Raises: + ValueError: As for ``SideEffectNode.from_descriptor``. + """ + return _LoadedSideEffectJobNode(descriptor, input_stream) + # ------------------------------------------------------------------ # DB attachment # ------------------------------------------------------------------ @@ -1280,3 +1335,293 @@ async def process_one(tag: TagProtocol, data: DataProtocol) -> None: self._hold(passed_on) finally: await output.close() + + +# --------------------------------------------------------------------------- +# Side-effect nodes restored from a saved file (ITL-1094) +# --------------------------------------------------------------------------- + + +class _SavedSideEffectNode: + """What a saved pipeline or job file records for one side-effect node. + + The file holds the node's label, content hash, pipeline hash, output schema, node + uri, data context and Orcapod configuration; it holds neither the side-effect + function nor the pod configuration. + + Args: + descriptor: The node's entry in the saved file. + + Raises: + ValueError: If the descriptor lacks ``content_hash``, ``pipeline_hash`` or + ``node_uri``. + """ + + def __init__(self, descriptor: dict[str, Any]) -> None: + from orcapod.config import DEFAULT_CONFIG, OrcapodConfig + from orcapod.contexts import resolve_saved_context + from orcapod.types import ContentHash + + missing = [ + name + for name in ("content_hash", "pipeline_hash", "node_uri") + if not descriptor.get(name) + ] + if missing: + raise ValueError( + f"The saved side-effect node {descriptor.get('label')!r} has no " + f"{', '.join(missing)}." + ) + self.descriptor = descriptor + self.label: str | None = descriptor.get("label") + self.content_hash = ContentHash.from_string(descriptor["content_hash"]) + self.pipeline_hash = ContentHash.from_string(descriptor["pipeline_hash"]) + self.node_uri: tuple[str, ...] = tuple(descriptor["node_uri"]) + self.data_context = resolve_saved_context(descriptor.get("data_context_key")) + config_dict = descriptor.get("config") + self.orcapod_config = ( + OrcapodConfig.from_dict(config_dict) + if config_dict is not None + else DEFAULT_CONFIG + ) + self._output_schema: dict[str, Any] = descriptor.get("output_schema") or {} + + def output_schema(self) -> tuple[Schema, Schema]: + """The saved output schema (every column ``T`` when nullability is unrecorded). + + Raises: + ValueError: If a column's saved type cannot be restored to a Python type. + """ + from orcapod.pipeline.serialization import deserialize_output_schema + + return deserialize_output_schema( + self._output_schema, self.data_context.type_converter + ) + + def keys(self) -> tuple[tuple[str, ...], tuple[str, ...]]: + """The saved tag and data column names.""" + return ( + tuple(self._output_schema.get("tag", {})), + tuple(self._output_schema.get("data", {})), + ) + + +def _cannot_deliver_error(label: str | None) -> RuntimeError: + """The error for asking a loaded side-effect node to deliver.""" + return RuntimeError( + f"Side-effect node {label!r} was loaded from a saved file, which holds no " + "side-effect function, so it cannot deliver. Build the job from its code to " + "run its side effects." + ) + + +class _LoadedSideEffectNode(SideEffectNode): + """A side-effect node of a loaded pipeline blueprint: no pod, never delivers. + + Built by ``SideEffectNode.from_descriptor``. Every method that would use the pod + reads the saved record instead, or raises. + + Args: + descriptor: The node's entry in the saved file. + input_stream: The restored node upstream of it. + """ + + def __init__( + self, descriptor: dict[str, Any], input_stream: StreamProtocol + ) -> None: + saved = _SavedSideEffectNode(descriptor) + super().__init__( + # A loaded node has no pod: every method that would use it is overridden. + side_effect_pod=None, # pyright: ignore[reportArgumentType] + input_stream=input_stream, + label=saved.label, + ) + self._saved = saved + self._descriptor = descriptor + self._data_context = saved.data_context + self._orcapod_config = saved.orcapod_config + + @property + @override + def producer(self) -> SideEffectPod | None: + """None: a loaded node has no pod.""" + return None + + @override + def identity_structure(self) -> Any: + """The saved content hash: what the saved node's identity hashed to.""" + return self._saved.content_hash + + @override + def pipeline_identity_structure(self) -> Any: + """The saved pipeline hash.""" + return self._saved.pipeline_hash + + @override + def content_hash(self, hasher=None) -> ContentHash: + """The saved content hash.""" + return self._saved.content_hash + + @override + def pipeline_hash(self, hasher=None) -> ContentHash: + """The saved pipeline hash.""" + return self._saved.pipeline_hash + + @override + def computed_label(self) -> str | None: + return None + + @override + def output_schema( + self, + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> tuple[Schema, Schema]: + """The saved output schema.""" + return self._saved.output_schema() + + @override + def keys( + self, + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> tuple[tuple[str, ...], tuple[str, ...]]: + """The saved column names.""" + return self._saved.keys() + + @override + def _rows_data_context(self) -> DataContext: + return self._saved.data_context + + @property + @override + def node_uri(self) -> tuple[str, ...]: + """The saved node uri.""" + return self._saved.node_uri + + @override + def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: + """Raise: a blueprint node delivers when read, and this one cannot deliver.""" + raise _cannot_deliver_error(self.label) + + +class _LoadedSideEffectJobNode(SideEffectJobNode): + """A side-effect node of a loaded job: no pod, never delivers (ITL-1094). + + Built by ``SideEffectJobNode.from_descriptor``. Its reads are those of every + ``SideEffectJobNode``: the rows its last run in this process passed on, and with + none, no rows. It cannot run: ``execute``, ``async_execute`` and ``run()`` raise + before anything is read, delivered or logged. + + Args: + descriptor: The node's entry in the saved file. + input_stream: The restored job node upstream of it. + """ + + def __init__( + self, descriptor: dict[str, Any], input_stream: StreamProtocol + ) -> None: + saved = _SavedSideEffectNode(descriptor) + super().__init__( + # A loaded node has no pod: every method that would use it is overridden. + side_effect_pod=None, # pyright: ignore[reportArgumentType] + input_stream=input_stream, + label=saved.label, + ) + self._saved = saved + self._descriptor = descriptor + self._data_context = saved.data_context + self._orcapod_config = saved.orcapod_config + + @property + @override + def producer(self) -> SideEffectPod | None: + """None: a loaded node has no pod.""" + return None + + @override + def identity_structure(self) -> Any: + """The saved content hash: what the saved node's identity hashed to.""" + return self._saved.content_hash + + @override + def pipeline_identity_structure(self) -> Any: + """The saved pipeline hash.""" + return self._saved.pipeline_hash + + @override + def content_hash(self, hasher=None) -> ContentHash: + """The saved content hash.""" + return self._saved.content_hash + + @override + def pipeline_hash(self, hasher=None) -> ContentHash: + """The saved pipeline hash: it names the saved run's invocation log.""" + return self._saved.pipeline_hash + + @override + def computed_label(self) -> str | None: + return None + + @override + def output_schema( + self, + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> tuple[Schema, Schema]: + """The saved output schema.""" + return self._saved.output_schema() + + @override + def keys( + self, + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> tuple[tuple[str, ...], tuple[str, ...]]: + """The saved column names.""" + return self._saved.keys() + + @override + def _rows_data_context(self) -> DataContext: + return self._saved.data_context + + @property + @override + def node_uri(self) -> tuple[str, ...]: + """The saved node uri.""" + return self._saved.node_uri + + @override + def execute( + self, + input_stream: StreamProtocol, + *, + observer: ExecutionObserverProtocol | None = None, + run_id: str | None = None, + ) -> list[tuple[TagProtocol, DataProtocol]]: + """Raise: the node has no side-effect function.""" + raise _cannot_deliver_error(self.label) + + @override + async def async_execute( + self, + inputs: Sequence[ReadableChannel[tuple[TagProtocol, DataProtocol]]], + output: WritableChannel[tuple[TagProtocol, DataProtocol]], + *, + observer: ExecutionObserverProtocol | None = None, + run_id: str | None = None, + ) -> None: + """Close ``output`` and raise: the node has no side-effect function.""" + try: + raise _cannot_deliver_error(self.label) + finally: + await output.close() + + @override + def run(self) -> StreamView: + """Raise: the node has no side-effect function.""" + raise _cannot_deliver_error(self.label) diff --git a/tests/test_pipeline/loaded_job/test_loaded_side_effect_nodes.py b/tests/test_pipeline/loaded_job/test_loaded_side_effect_nodes.py new file mode 100644 index 000000000..8ac76d286 --- /dev/null +++ b/tests/test_pipeline/loaded_job/test_loaded_side_effect_nodes.py @@ -0,0 +1,234 @@ +"""A loaded job restores its side-effect nodes, read-only (ITL-1094). + +A saved job file records each side-effect node (``node_type`` ``"side_effect"``) but no +side-effect function and no pod configuration. ``PipelineJob.load`` and ``Pipeline.load`` +restore the node with its saved label, content hash, pipeline hash, output schema and +position in the graph. It never delivers: a read returns the rows of a run in this process +(none), and every run raises. The nodes below it keep their saved identity. +""" + +from __future__ import annotations + +import asyncio +import json +import pathlib +from collections.abc import Callable +from typing import Any + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.databases import DeltaTableDatabase +from orcapod.pipeline import Pipeline +from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.side_effects import SideEffectPod, SideEffectPodConfig +from orcapod.sources import DictSource +from orcapod.streams import ViewStatus + +ROWS = 3 +SIDE_EFFECT_NODES = ("delivered", "tail") + + +def _source() -> DictSource: + """Three rows: tag ``k``, data ``x``.""" + return DictSource( + [{"k": i, "x": i} for i in range(ROWS)], tag_columns=["k"], source_id="s" + ) + + +def _saved_job(root: pathlib.Path) -> tuple[PipelineJob, list[int], pathlib.Path]: + """Run and save ``s -> delivered -> down -> tail``; return the job, deliveries, file. + + ``delivered`` (a side-effect node in the middle, ``track_completion`` and + ``on_error="log"``) passes ``x`` on to the function node ``down`` (``y = x + 100``), + and ``tail`` (a side-effect node at the end) receives ``y``. + """ + sent: list[int] = [] + + def deliver(ctx: Any, x: int) -> None: + """Record ``x``.""" + sent.append(x) + + def notify(ctx: Any, y: int) -> None: + """Record ``y``.""" + sent.append(y) + + @function_pod(output_keys="y") + def plus(x: int) -> int: + """``x + 100``.""" + return x + 100 + + job = PipelineJob(name="se", database=DeltaTableDatabase(root / "store")) + middle = SideEffectPod( + deliver, config=SideEffectPodConfig(track_completion=True, on_error="log") + ) + end = SideEffectPod(notify, config=SideEffectPodConfig(track_completion=False)) + with job: + delivered = middle(_source(), label="delivered") + down = plus.pod(delivered, label="down") + _ = end(down, label="tail") + _ = job.run() + assert len(sent) == 2 * ROWS + path = root / "job.json" + job.save(path) + return job, sent, path + + +@pytest.fixture +def saved(tmp_path: pathlib.Path) -> tuple[PipelineJob, list[int], pathlib.Path]: + """A job with two side-effect nodes, run and saved.""" + return _saved_job(tmp_path) + + +def _descriptors(path: pathlib.Path) -> dict[str, dict[str, Any]]: + """The node entries of the job file at ``path``, by label.""" + nodes = json.loads(path.read_text())["pipeline"]["nodes"] + return {descriptor["label"]: descriptor for descriptor in nodes.values()} + + +def _reads(node: Any) -> dict[str, Callable[[], Any]]: + """Every public read of ``node``, by name.""" + + async def collect() -> list[Any]: + return [row async for row in node.async_iter_data()] + + return { + "head": lambda: node.head(100), + "preview": node._repr_html_, + "iter_data": lambda: list(node.iter_data()), + "__iter__": lambda: list(node), + "async_iter_data": lambda: asyncio.run(collect()), + "as_table": node.as_table, + "as_polars_df": node.as_polars_df, + "as_df": node.as_df, + "as_lazy_frame": lambda: node.as_lazy_frame().collect(), + "as_pandas_df": node.as_pandas_df, + "flow": node.flow, + } + + +def test_a_loaded_job_lists_its_side_effect_nodes( + saved: tuple[PipelineJob, list[int], pathlib.Path], +) -> None: + """The node in the middle and the node at the end are both restored.""" + job, _, path = saved + loaded = PipelineJob.load(path) + assert set(loaded.nodes) == set(job.nodes) == {"s", "delivered", "down", "tail"} + for label in SIDE_EFFECT_NODES: + assert loaded.nodes[label].label == label + + +def test_a_restored_side_effect_node_has_its_saved_identity_and_position( + saved: tuple[PipelineJob, list[int], pathlib.Path], +) -> None: + """Saved content hash and pipeline hash; upstream is the restored node above it.""" + job, _, path = saved + descriptors = _descriptors(path) + loaded = PipelineJob.load(path) + for label in SIDE_EFFECT_NODES: + node = loaded.nodes[label] + assert node.content_hash().to_string() == descriptors[label]["content_hash"] + # The pipeline hash names the invocation log: the run's, unchanged. + assert node.pipeline_hash() == job.nodes[label].pipeline_hash() + assert node.output_schema() == job.nodes[label].output_schema() + assert node.keys() == job.nodes[label].keys() + assert loaded.nodes["delivered"].upstreams == (loaded.nodes["s"],) + assert loaded.nodes["tail"].upstreams == (loaded.nodes["down"],) + + +def test_the_node_below_keeps_its_identity_and_its_upstream( + saved: tuple[PipelineJob, list[int], pathlib.Path], +) -> None: + """``down`` hangs below the restored ``delivered`` with the run's hashes.""" + job, _, path = saved + loaded = PipelineJob.load(path) + down = loaded.nodes["down"] + assert down.upstreams == (loaded.nodes["delivered"],) + assert down.content_hash() == job.nodes["down"].content_hash() + assert down.pipeline_hash() == job.nodes["down"].pipeline_hash() + + +def test_no_read_or_run_of_a_loaded_job_delivers( + saved: tuple[PipelineJob, list[int], pathlib.Path], +) -> None: + """Reads return no rows, runs raise, and nothing is delivered.""" + _, sent, path = saved + delivered_before = list(sent) + loaded = PipelineJob.load(path) + for label in SIDE_EFFECT_NODES: + node = loaded.nodes[label] + for name, read in _reads(node).items(): + _ = read() + assert sent == delivered_before, f"{label}.{name} delivered" + assert list(node.iter_data()) == [] + assert node.as_table().num_rows == 0 + assert node.head(100).status is ViewStatus.NOT_COMPUTED + with pytest.raises(RuntimeError, match="cannot deliver"): + _ = node.run() + with pytest.raises(RuntimeError): + _ = loaded.run() + with pytest.raises(RuntimeError): + _ = loaded.run(orchestrator=AsyncPipelineOrchestrator()) + assert sent == delivered_before + + +def test_saving_a_loaded_job_keeps_its_side_effect_nodes( + saved: tuple[PipelineJob, list[int], pathlib.Path], tmp_path: pathlib.Path +) -> None: + """Their entries and every edge are written back unchanged; loading again keeps them.""" + _, _, path = saved + again = tmp_path / "again.json" + PipelineJob.load(path).save(again) + first = json.loads(path.read_text())["pipeline"] + second = json.loads(again.read_text())["pipeline"] + for key, descriptor in first["nodes"].items(): + if descriptor["node_type"] == "side_effect": + assert second["nodes"][key] == descriptor + assert set(second["nodes"]) == set(first["nodes"]) + assert sorted(map(tuple, second["edges"])) == sorted(map(tuple, first["edges"])) + assert set(PipelineJob.load(again).nodes) == {"s", "delivered", "down", "tail"} + + +def test_pipeline_load_restores_side_effect_nodes( + saved: tuple[PipelineJob, list[int], pathlib.Path], tmp_path: pathlib.Path +) -> None: + """The pipeline blueprint lists them too, with the saved identity.""" + job, sent, _ = saved + delivered_before = list(sent) + blueprint = tmp_path / "pipeline.json" + job.pipeline.save(blueprint) + saved_nodes = json.loads(blueprint.read_text())["nodes"] + by_label = {descriptor["label"]: descriptor for descriptor in saved_nodes.values()} + loaded = Pipeline.load(blueprint) + assert set(loaded.nodes) == {"s", "delivered", "down", "tail"} + for label in SIDE_EFFECT_NODES: + node = loaded.nodes[label] + assert node.content_hash().to_string() == by_label[label]["content_hash"] + assert node.pipeline_hash().to_string() == by_label[label]["pipeline_hash"] + with pytest.raises(RuntimeError, match="cannot deliver"): + _ = node.as_table() + assert sent == delivered_before + + +def test_a_file_without_recorded_nullability_restores_side_effect_nodes( + saved: tuple[PipelineJob, list[int], pathlib.Path], tmp_path: pathlib.Path +) -> None: + """A job file of format 0.1.0 (v0.2: no ``nullable`` record) restores them too.""" + _, _, path = saved + data = json.loads(path.read_text()) + data["orcapod_pipeline_job_version"] = "0.1.0" + data["pipeline"]["orcapod_pipeline_version"] = "0.1.0" + for descriptor in data["pipeline"]["nodes"].values(): + del descriptor["output_schema"]["nullable"] + descriptor.get("source_config", {}).pop("nullable", None) + old = tmp_path / "format_0_1_0.json" + old.write_text(json.dumps(data)) + + loaded = PipelineJob.load(old) + assert set(loaded.nodes) == {"s", "delivered", "down", "tail"} + assert loaded.nodes["delivered"].keys() == (("k",), ("x",)) + # Saving it again keeps the nullability unrecorded, as for every other node. + again = tmp_path / "again.json" + loaded.save(again) + for descriptor in _descriptors(again).values(): + assert "nullable" not in descriptor["output_schema"] From 2dd32eaa2736a2ccd23df4bdd50485d9b85121fc Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 01:56:07 -0800 Subject: [PATCH 32/48] fix(pipeline): restore loaded side-effect nodes from their own module (ITL-1094) (WIP) Co-Authored-By: Claude Opus 5.5 --- src/orcapod/pipeline/graph.py | 6 +- src/orcapod/pipeline/job.py | 5 +- src/orcapod/pipeline/loaded_side_effects.py | 327 +++++++++++++++++++ src/orcapod/side_effects.py | 339 -------------------- 4 files changed, 336 insertions(+), 341 deletions(-) create mode 100644 src/orcapod/pipeline/loaded_side_effects.py diff --git a/src/orcapod/pipeline/graph.py b/src/orcapod/pipeline/graph.py index a8795653e..dfb07ddd2 100644 --- a/src/orcapod/pipeline/graph.py +++ b/src/orcapod/pipeline/graph.py @@ -467,6 +467,8 @@ def load(cls, path: str | Path) -> "Pipeline": reconstructed[node_hash] = node elif node_type == "side_effect": + from orcapod.pipeline.loaded_side_effects import LoadedSideEffectNode + # Restored read-only: the file holds no side-effect function (ITL-1094). up_hashes = upstream_map.get(node_hash, []) upstream_node = reconstructed.get(up_hashes[0]) if up_hashes else None @@ -475,7 +477,9 @@ def load(cls, path: str | Path) -> "Pipeline": f"The saved side-effect node {descriptor.get('label')!r} has " "no upstream node in the file." ) - reconstructed[node_hash] = SideEffectNode.from_descriptor( + # GraphNode predates side-effect nodes; compile() stores them in the + # same maps through the untyped side_effect_node_class. + reconstructed[node_hash] = LoadedSideEffectNode( # pyright: ignore[reportArgumentType] descriptor, input_stream=upstream_node ) diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 45c4e58b8..33cd18875 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -1551,6 +1551,7 @@ def load( from orcapod.core.nodes.function_node import FunctionNode as _FunctionNode from orcapod.core.nodes.operator_node import OperatorNode as _OperatorNode from orcapod.core.nodes.source_node import SourceNode as _SourceNode + from orcapod.pipeline.loaded_side_effects import LoadedSideEffectJobNode job._sources = dict(sources) @@ -1627,7 +1628,9 @@ def load( # Read-only: it never delivers (ITL-1094). Its upstream is the restored # job node, so the nodes below it keep their position in the graph. preds = list(job._hash_graph.predecessors(node_hash)) - job._persistent_node_map[node_hash] = SideEffectJobNode.from_descriptor( + # JobNode predates side-effect nodes; compile() stores them in this map + # too, through the untyped side_effect_node_class. + job._persistent_node_map[node_hash] = LoadedSideEffectJobNode( # pyright: ignore[reportArgumentType] getattr(bp_node, "_descriptor", None) or {}, input_stream=job._persistent_node_map[preds[0]], ) diff --git a/src/orcapod/pipeline/loaded_side_effects.py b/src/orcapod/pipeline/loaded_side_effects.py new file mode 100644 index 000000000..9a887dfbf --- /dev/null +++ b/src/orcapod/pipeline/loaded_side_effects.py @@ -0,0 +1,327 @@ +"""Side-effect nodes restored from a saved pipeline or job file, read-only (ITL-1094). + +A saved file records a side-effect node's label, content hash, pipeline hash, output +schema, node uri, data context and Orcapod configuration. It records neither the +side-effect function nor the pod configuration (``on_error``, ``track_completion``, +``drop_on_failure``, ``max_concurrency``). ``Pipeline.load`` and ``PipelineJob.load`` +restore the node from that record, in its saved place in the graph, with no pod: it +never delivers and never writes its invocation log. +""" + +from __future__ import annotations + +from collections.abc import Iterator, Sequence +from typing import TYPE_CHECKING, Any + +from typing_extensions import override + +from orcapod.side_effects import SideEffectJobNode, SideEffectNode + +if TYPE_CHECKING: + from orcapod.channels import ReadableChannel, WritableChannel + from orcapod.config import OrcapodConfig + from orcapod.contexts import DataContext + from orcapod.core.streams.view import StreamView + from orcapod.protocols.core_protocols import ( + DataProtocol, + StreamProtocol, + TagProtocol, + ) + from orcapod.protocols.observability_protocols import ExecutionObserverProtocol + from orcapod.side_effects import SideEffectPod + from orcapod.types import ColumnConfig, ContentHash, Schema + + +class SavedSideEffectNode: + """What a saved pipeline or job file records for one side-effect node. + + Args: + descriptor: The node's entry in the saved file (``node_type`` + ``"side_effect"``). + + Raises: + ValueError: If the descriptor lacks ``content_hash``, ``pipeline_hash`` or + ``node_uri``. + """ + + def __init__(self, descriptor: dict[str, Any]) -> None: + from orcapod.config import DEFAULT_CONFIG, OrcapodConfig + from orcapod.contexts import resolve_saved_context + from orcapod.types import ContentHash + + missing = [ + name + for name in ("content_hash", "pipeline_hash", "node_uri") + if not descriptor.get(name) + ] + if missing: + raise ValueError( + f"The saved side-effect node {descriptor.get('label')!r} has no " + f"{', '.join(missing)}." + ) + self.descriptor: dict[str, Any] = descriptor + self.label: str | None = descriptor.get("label") + self.content_hash: ContentHash = ContentHash.from_string( + descriptor["content_hash"] + ) + self.pipeline_hash: ContentHash = ContentHash.from_string( + descriptor["pipeline_hash"] + ) + self.node_uri: tuple[str, ...] = tuple(descriptor["node_uri"]) + self.data_context: DataContext = resolve_saved_context( + descriptor.get("data_context_key") + ) + config_dict = descriptor.get("config") + self.orcapod_config: OrcapodConfig = ( + OrcapodConfig.from_dict(config_dict) + if config_dict is not None + else DEFAULT_CONFIG + ) + self._output_schema: dict[str, Any] = descriptor.get("output_schema") or {} + + def output_schema(self) -> tuple[Schema, Schema]: + """The saved output schema; every column ``T`` when nullability is unrecorded. + + Raises: + ValueError: If a column's saved type cannot be restored to a Python type. + """ + from orcapod.pipeline.serialization import deserialize_output_schema + + return deserialize_output_schema( + self._output_schema, self.data_context.type_converter + ) + + def keys(self) -> tuple[tuple[str, ...], tuple[str, ...]]: + """The saved tag and data column names.""" + return ( + tuple(self._output_schema.get("tag", {})), + tuple(self._output_schema.get("data", {})), + ) + + +def cannot_deliver_error(label: str | None) -> RuntimeError: + """Return the error for asking a loaded side-effect node to deliver.""" + return RuntimeError( + f"Side-effect node {label!r} was loaded from a saved file, which holds no " + "side-effect function, so it cannot deliver. Build the job from its code to " + "run its side effects." + ) + + +class LoadedSideEffectNode(SideEffectNode): + """A side-effect node of a loaded pipeline blueprint: no pod, never delivers. + + Built by ``Pipeline.load``. Its identity, schema and node uri are the saved ones. + A blueprint side-effect node delivers when it is read, so every read raises. + + Args: + descriptor: The node's entry in the saved file. + input_stream: The restored node upstream of it. + + Raises: + ValueError: As for ``SavedSideEffectNode``. + """ + + def __init__( + self, descriptor: dict[str, Any], input_stream: StreamProtocol + ) -> None: + saved = SavedSideEffectNode(descriptor) + super().__init__( + # A loaded node has no pod: every method that would use it is overridden. + side_effect_pod=None, # pyright: ignore[reportArgumentType] + input_stream=input_stream, + label=saved.label, + ) + self._saved: SavedSideEffectNode = saved + self._descriptor: dict[str, Any] = descriptor + self._data_context = saved.data_context + self._orcapod_config = saved.orcapod_config + + @property + @override + def producer(self) -> SideEffectPod | None: + """``None``: a loaded node has no pod.""" + return None + + @override + def identity_structure(self) -> Any: + """The saved content hash.""" + return self._saved.content_hash + + @override + def pipeline_identity_structure(self) -> Any: + """The saved pipeline hash.""" + return self._saved.pipeline_hash + + @override + def content_hash(self, hasher: Any = None) -> ContentHash: + """The saved content hash.""" + return self._saved.content_hash + + @override + def pipeline_hash(self, hasher: Any = None) -> ContentHash: + """The saved pipeline hash.""" + return self._saved.pipeline_hash + + @override + def computed_label(self) -> str | None: + return None + + @override + def output_schema( + self, + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> tuple[Schema, Schema]: + """The saved output schema.""" + return self._saved.output_schema() + + @override + def keys( + self, + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> tuple[tuple[str, ...], tuple[str, ...]]: + """The saved column names.""" + return self._saved.keys() + + @override + def _rows_data_context(self) -> DataContext: + return self._saved.data_context + + @property + @override + def node_uri(self) -> tuple[str, ...]: + """The saved node uri.""" + return self._saved.node_uri + + @override + def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: + """Raise: a blueprint node delivers when read, and this one cannot deliver.""" + raise cannot_deliver_error(self.label) + + +class LoadedSideEffectJobNode(SideEffectJobNode): + """A side-effect node of a loaded job: no pod, never delivers. + + Built by ``PipelineJob.load``. Its identity, schema and node uri are the saved + ones; the pipeline hash names the saved run's invocation log, which the node never + reads or writes. Its reads are those of every ``SideEffectJobNode``: the rows its + last run in this process passed on, so none. ``execute``, ``async_execute`` and + ``run()`` raise before anything is read, delivered or logged. + + Args: + descriptor: The node's entry in the saved file. + input_stream: The restored job node upstream of it. + + Raises: + ValueError: As for ``SavedSideEffectNode``. + """ + + def __init__( + self, descriptor: dict[str, Any], input_stream: StreamProtocol + ) -> None: + saved = SavedSideEffectNode(descriptor) + super().__init__( + # A loaded node has no pod: every method that would use it is overridden. + side_effect_pod=None, # pyright: ignore[reportArgumentType] + input_stream=input_stream, + label=saved.label, + ) + self._saved: SavedSideEffectNode = saved + self._descriptor: dict[str, Any] = descriptor + self._data_context = saved.data_context + self._orcapod_config = saved.orcapod_config + + @property + @override + def producer(self) -> SideEffectPod | None: + """``None``: a loaded node has no pod.""" + return None + + @override + def identity_structure(self) -> Any: + """The saved content hash.""" + return self._saved.content_hash + + @override + def pipeline_identity_structure(self) -> Any: + """The saved pipeline hash.""" + return self._saved.pipeline_hash + + @override + def content_hash(self, hasher: Any = None) -> ContentHash: + """The saved content hash.""" + return self._saved.content_hash + + @override + def pipeline_hash(self, hasher: Any = None) -> ContentHash: + """The saved pipeline hash: it names the saved run's invocation log.""" + return self._saved.pipeline_hash + + @override + def computed_label(self) -> str | None: + return None + + @override + def output_schema( + self, + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> tuple[Schema, Schema]: + """The saved output schema.""" + return self._saved.output_schema() + + @override + def keys( + self, + *, + columns: ColumnConfig | dict[str, Any] | None = None, + all_info: bool = False, + ) -> tuple[tuple[str, ...], tuple[str, ...]]: + """The saved column names.""" + return self._saved.keys() + + @override + def _rows_data_context(self) -> DataContext: + return self._saved.data_context + + @property + @override + def node_uri(self) -> tuple[str, ...]: + """The saved node uri.""" + return self._saved.node_uri + + @override + def execute( + self, + input_stream: StreamProtocol, + *, + observer: ExecutionObserverProtocol | None = None, + run_id: str | None = None, + ) -> list[tuple[TagProtocol, DataProtocol]]: + """Raise: the node has no side-effect function.""" + raise cannot_deliver_error(self.label) + + @override + async def async_execute( + self, + inputs: Sequence[ReadableChannel[tuple[TagProtocol, DataProtocol]]], + output: WritableChannel[tuple[TagProtocol, DataProtocol]], + *, + observer: ExecutionObserverProtocol | None = None, + run_id: str | None = None, + ) -> None: + """Close ``output`` and raise: the node has no side-effect function.""" + try: + raise cannot_deliver_error(self.label) + finally: + await output.close() + + @override + def run(self) -> StreamView: + """Raise: the node has no side-effect function.""" + raise cannot_deliver_error(self.label) diff --git a/src/orcapod/side_effects.py b/src/orcapod/side_effects.py index 180fee56f..db545c08a 100644 --- a/src/orcapod/side_effects.py +++ b/src/orcapod/side_effects.py @@ -915,31 +915,6 @@ def node_uri(self) -> tuple[str, ...]: """ return self._pod.uri - @classmethod - def from_descriptor( - cls, descriptor: dict[str, Any], input_stream: StreamProtocol - ) -> SideEffectNode: - """Restore a side-effect node of a saved pipeline, read-only (ITL-1094). - - The saved file holds no side-effect function and no pod configuration, so the - node cannot deliver: every read or run that would deliver raises. Its label, - content hash, pipeline hash, output schema, node uri, data context and Orcapod - configuration are the saved ones. - - Args: - descriptor: The node's entry in the saved file (``node_type`` - ``"side_effect"``). - input_stream: The restored node upstream of it. - - Returns: - The read-only node. - - Raises: - ValueError: If the descriptor lacks ``content_hash``, ``pipeline_hash`` or - ``node_uri``. - """ - return _LoadedSideEffectNode(descriptor, input_stream) - # --------------------------------------------------------------------------- # SideEffectJobNode — DB-backed execution node @@ -1028,30 +1003,6 @@ def _refuse_log_of_another_data_context(self) -> None: raise self._current_log_checked = True - @classmethod - @override - def from_descriptor( - cls, descriptor: dict[str, Any], input_stream: StreamProtocol - ) -> SideEffectJobNode: - """Restore a side-effect node of a saved job, read-only (ITL-1094). - - Used by ``PipelineJob.load``. The node has no side-effect function: it never - delivers and never writes its invocation log. Its reads follow every - ``SideEffectJobNode``: with no rows from a run in this process, none. - ``execute``, ``async_execute`` and ``run()`` raise. - - Args: - descriptor: The node's entry in the saved file. - input_stream: The restored job node upstream of it. - - Returns: - The read-only job node. - - Raises: - ValueError: As for ``SideEffectNode.from_descriptor``. - """ - return _LoadedSideEffectJobNode(descriptor, input_stream) - # ------------------------------------------------------------------ # DB attachment # ------------------------------------------------------------------ @@ -1335,293 +1286,3 @@ async def process_one(tag: TagProtocol, data: DataProtocol) -> None: self._hold(passed_on) finally: await output.close() - - -# --------------------------------------------------------------------------- -# Side-effect nodes restored from a saved file (ITL-1094) -# --------------------------------------------------------------------------- - - -class _SavedSideEffectNode: - """What a saved pipeline or job file records for one side-effect node. - - The file holds the node's label, content hash, pipeline hash, output schema, node - uri, data context and Orcapod configuration; it holds neither the side-effect - function nor the pod configuration. - - Args: - descriptor: The node's entry in the saved file. - - Raises: - ValueError: If the descriptor lacks ``content_hash``, ``pipeline_hash`` or - ``node_uri``. - """ - - def __init__(self, descriptor: dict[str, Any]) -> None: - from orcapod.config import DEFAULT_CONFIG, OrcapodConfig - from orcapod.contexts import resolve_saved_context - from orcapod.types import ContentHash - - missing = [ - name - for name in ("content_hash", "pipeline_hash", "node_uri") - if not descriptor.get(name) - ] - if missing: - raise ValueError( - f"The saved side-effect node {descriptor.get('label')!r} has no " - f"{', '.join(missing)}." - ) - self.descriptor = descriptor - self.label: str | None = descriptor.get("label") - self.content_hash = ContentHash.from_string(descriptor["content_hash"]) - self.pipeline_hash = ContentHash.from_string(descriptor["pipeline_hash"]) - self.node_uri: tuple[str, ...] = tuple(descriptor["node_uri"]) - self.data_context = resolve_saved_context(descriptor.get("data_context_key")) - config_dict = descriptor.get("config") - self.orcapod_config = ( - OrcapodConfig.from_dict(config_dict) - if config_dict is not None - else DEFAULT_CONFIG - ) - self._output_schema: dict[str, Any] = descriptor.get("output_schema") or {} - - def output_schema(self) -> tuple[Schema, Schema]: - """The saved output schema (every column ``T`` when nullability is unrecorded). - - Raises: - ValueError: If a column's saved type cannot be restored to a Python type. - """ - from orcapod.pipeline.serialization import deserialize_output_schema - - return deserialize_output_schema( - self._output_schema, self.data_context.type_converter - ) - - def keys(self) -> tuple[tuple[str, ...], tuple[str, ...]]: - """The saved tag and data column names.""" - return ( - tuple(self._output_schema.get("tag", {})), - tuple(self._output_schema.get("data", {})), - ) - - -def _cannot_deliver_error(label: str | None) -> RuntimeError: - """The error for asking a loaded side-effect node to deliver.""" - return RuntimeError( - f"Side-effect node {label!r} was loaded from a saved file, which holds no " - "side-effect function, so it cannot deliver. Build the job from its code to " - "run its side effects." - ) - - -class _LoadedSideEffectNode(SideEffectNode): - """A side-effect node of a loaded pipeline blueprint: no pod, never delivers. - - Built by ``SideEffectNode.from_descriptor``. Every method that would use the pod - reads the saved record instead, or raises. - - Args: - descriptor: The node's entry in the saved file. - input_stream: The restored node upstream of it. - """ - - def __init__( - self, descriptor: dict[str, Any], input_stream: StreamProtocol - ) -> None: - saved = _SavedSideEffectNode(descriptor) - super().__init__( - # A loaded node has no pod: every method that would use it is overridden. - side_effect_pod=None, # pyright: ignore[reportArgumentType] - input_stream=input_stream, - label=saved.label, - ) - self._saved = saved - self._descriptor = descriptor - self._data_context = saved.data_context - self._orcapod_config = saved.orcapod_config - - @property - @override - def producer(self) -> SideEffectPod | None: - """None: a loaded node has no pod.""" - return None - - @override - def identity_structure(self) -> Any: - """The saved content hash: what the saved node's identity hashed to.""" - return self._saved.content_hash - - @override - def pipeline_identity_structure(self) -> Any: - """The saved pipeline hash.""" - return self._saved.pipeline_hash - - @override - def content_hash(self, hasher=None) -> ContentHash: - """The saved content hash.""" - return self._saved.content_hash - - @override - def pipeline_hash(self, hasher=None) -> ContentHash: - """The saved pipeline hash.""" - return self._saved.pipeline_hash - - @override - def computed_label(self) -> str | None: - return None - - @override - def output_schema( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[Schema, Schema]: - """The saved output schema.""" - return self._saved.output_schema() - - @override - def keys( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[tuple[str, ...], tuple[str, ...]]: - """The saved column names.""" - return self._saved.keys() - - @override - def _rows_data_context(self) -> DataContext: - return self._saved.data_context - - @property - @override - def node_uri(self) -> tuple[str, ...]: - """The saved node uri.""" - return self._saved.node_uri - - @override - def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: - """Raise: a blueprint node delivers when read, and this one cannot deliver.""" - raise _cannot_deliver_error(self.label) - - -class _LoadedSideEffectJobNode(SideEffectJobNode): - """A side-effect node of a loaded job: no pod, never delivers (ITL-1094). - - Built by ``SideEffectJobNode.from_descriptor``. Its reads are those of every - ``SideEffectJobNode``: the rows its last run in this process passed on, and with - none, no rows. It cannot run: ``execute``, ``async_execute`` and ``run()`` raise - before anything is read, delivered or logged. - - Args: - descriptor: The node's entry in the saved file. - input_stream: The restored job node upstream of it. - """ - - def __init__( - self, descriptor: dict[str, Any], input_stream: StreamProtocol - ) -> None: - saved = _SavedSideEffectNode(descriptor) - super().__init__( - # A loaded node has no pod: every method that would use it is overridden. - side_effect_pod=None, # pyright: ignore[reportArgumentType] - input_stream=input_stream, - label=saved.label, - ) - self._saved = saved - self._descriptor = descriptor - self._data_context = saved.data_context - self._orcapod_config = saved.orcapod_config - - @property - @override - def producer(self) -> SideEffectPod | None: - """None: a loaded node has no pod.""" - return None - - @override - def identity_structure(self) -> Any: - """The saved content hash: what the saved node's identity hashed to.""" - return self._saved.content_hash - - @override - def pipeline_identity_structure(self) -> Any: - """The saved pipeline hash.""" - return self._saved.pipeline_hash - - @override - def content_hash(self, hasher=None) -> ContentHash: - """The saved content hash.""" - return self._saved.content_hash - - @override - def pipeline_hash(self, hasher=None) -> ContentHash: - """The saved pipeline hash: it names the saved run's invocation log.""" - return self._saved.pipeline_hash - - @override - def computed_label(self) -> str | None: - return None - - @override - def output_schema( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[Schema, Schema]: - """The saved output schema.""" - return self._saved.output_schema() - - @override - def keys( - self, - *, - columns: ColumnConfig | dict[str, Any] | None = None, - all_info: bool = False, - ) -> tuple[tuple[str, ...], tuple[str, ...]]: - """The saved column names.""" - return self._saved.keys() - - @override - def _rows_data_context(self) -> DataContext: - return self._saved.data_context - - @property - @override - def node_uri(self) -> tuple[str, ...]: - """The saved node uri.""" - return self._saved.node_uri - - @override - def execute( - self, - input_stream: StreamProtocol, - *, - observer: ExecutionObserverProtocol | None = None, - run_id: str | None = None, - ) -> list[tuple[TagProtocol, DataProtocol]]: - """Raise: the node has no side-effect function.""" - raise _cannot_deliver_error(self.label) - - @override - async def async_execute( - self, - inputs: Sequence[ReadableChannel[tuple[TagProtocol, DataProtocol]]], - output: WritableChannel[tuple[TagProtocol, DataProtocol]], - *, - observer: ExecutionObserverProtocol | None = None, - run_id: str | None = None, - ) -> None: - """Close ``output`` and raise: the node has no side-effect function.""" - try: - raise _cannot_deliver_error(self.label) - finally: - await output.close() - - @override - def run(self) -> StreamView: - """Raise: the node has no side-effect function.""" - raise _cannot_deliver_error(self.label) From e78960c8287820ed1dd5584b1316b629d06c7d76 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 01:57:46 -0800 Subject: [PATCH 33/48] docs: a loaded job keeps its side-effect nodes; changelog fragment and design issue (ITL-1094) Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1094.md | 12 ++++++++ design_issues/ITL-1094-LJ2.md | 28 +++++++++++++++++++ docs/concepts/function-pods.md | 7 +++++ ...-11-14-itl-1094-loaded-side-effect-node.md | 22 ++++++++++++++- 4 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 changelog.d/ITL-1094.md create mode 100644 design_issues/ITL-1094-LJ2.md diff --git a/changelog.d/ITL-1094.md b/changelog.d/ITL-1094.md new file mode 100644 index 000000000..dc0dba7fb --- /dev/null +++ b/changelog.d/ITL-1094.md @@ -0,0 +1,12 @@ +### Fixed + +- **A loaded job keeps its side-effect nodes.** `PipelineJob.load` and `Pipeline.load` + dropped every side-effect node of the saved file: `loaded.nodes["