diff --git a/.zed/rules b/.zed/rules index 59e335f4f..b706d13ed 100644 --- a/.zed/rules +++ b/.zed/rules @@ -843,7 +843,7 @@ data. The rules: non-stale run (for a full run: not load-only, nor anything above it) runs nothing; otherwise its non-current dependencies run first, each once, on the sync orchestrator (pipeline/node_run.py), and the run commits as job.run() does. A failed end-of-run - commit clears every node of the run. A load-only run's rows have status + commit clears only the nodes that ran; nodes it held keep their earlier run. A load-only run's rows have status loaded_only. Only job nodes' run() takes an argument (load_only). A loaded job (PipelineJob.load) runs only with load_only=True. job.run()/run_async() use the same state rule and plan (node_run.plan_run): a current node is diff --git a/CLAUDE.md b/CLAUDE.md index 4567f8ec4..836617c23 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -885,7 +885,7 @@ data. The rules: non-stale run (for a full run: not load-only, nor anything above it) runs nothing; otherwise its non-current dependencies run first, each once, on the sync orchestrator (`pipeline/node_run.py`), and the run commits as `job.run()` does. A failed end-of-run - commit clears every node of the run. A load-only run's rows have status + commit clears only the nodes that ran; nodes it held keep their earlier run. A load-only run's rows have status `loaded_only`. Only job nodes' `run()` takes an argument (`load_only`). A loaded job (`PipelineJob.load`) runs only with `load_only=True`. `job.run()`/`run_async()` use the same state rule and plan (`node_run.plan_run`): a current node is diff --git a/changelog.d/ITL-1121.md b/changelog.d/ITL-1121.md new file mode 100644 index 000000000..b0ff92bde --- /dev/null +++ b/changelog.d/ITL-1121.md @@ -0,0 +1,12 @@ +### Changed + +- A load-only run (`job.run(load_only=True)`, `run_async(load_only=True)`, `node.run(load_only=True)`, a source slot's `run(load_only=True)`) reads only job nodes, sources and streams that neither compute nor deliver (developer, November 14, 2026; ITL-1121). A pod's stream built outside the job is refused with a `ValueError` before anything is read: a function pod's stream (a cached pod's included), which computes when it is read, and a side-effect pod's stream or a `SideEffectNode` outside a job, which deliver their side effects (before, the run read them and delivered every row). An `ArrowTableStream` holds its rows and is read whatever its `upstreams` name. This applies whether a job node reads the stream, a source slot is bound to it or an operator's stream outside the job reads it. Build the pod inside the job, or run without `load_only`; a run without `load_only` is unchanged. ITL-1130 re-evaluates the choice. + +### Fixed + +- An observer that read a function job node in `on_node_end` emptied it: the node was marked current only after that hook, so the read found it stale and cleared the run's rows (ITL-1121). The node is now current before the hook, on sync, async and load-only runs. +- An operator job node without a database, or in `CacheMode.OFF`, whose output failed to compute in `run()` kept claiming a current run and its reads raised; it now reads as not run, like every node whose run raised. +- The run summary said "nothing to do; every node holds a current run" when the run ran a side-effect node (and delivered) or an operator node below a held function node. It now says so only when the run ran no node; `RunSummary` takes `ran_nothing`. +- A side-effect job node whose load-only `execute()` failed on its input stayed current, and so did the nodes below it; it is now cleared, as on the async path. +- An operator job node without a database, run with `load_only=True`, showed its rows as a full run's (`rows` instead of `loaded_only`) and a later full `run()` took them as its own. +- A side-effect job node over a stream outside a job, with a store attached, kept its rows as a current run when its `run()` was interrupted (`KeyboardInterrupt`) while committing the invocation log, so the next `run()` neither ran nor committed again; it is now cleared. diff --git a/design_issues/ITL-1121-NV1.md b/design_issues/ITL-1121-NV1.md new file mode 100644 index 000000000..85b0a8d50 --- /dev/null +++ b/design_issues/ITL-1121-NV1.md @@ -0,0 +1,39 @@ +## `src/orcapod/core/nodes/function_node.py` + +### NV1 — Job node view: a run's rows lost to an observer, failed runs left current, a misleading summary +**Status:** resolved +**Severity:** medium +**Issue:** ITL-1121 + +A full review of the job node current-run view (ITL-1095) found five defects, each reproduced +through the public interface (`superpowers/scripts/itl_1121/`): + +- A function job node was stamped as modified after `on_node_end`, so an observer reading the + node there found it stale and cleared the run's rows; the node read empty after the run (sync, + async and both load-only paths). +- An operator job node without a database, or in `CacheMode.OFF`, computed its lazy output while + building the `run()` view, outside the block that clears a failed run: after a failure it + claimed a current run and its reads raised. +- `RunSummary` said "nothing to do; every node holds a current run" when the run ran only a + side-effect node (which delivered) or an operator node below a held function node: the + headline judged only the function nodes' counts. +- A side-effect job node's sync `execute(load_only=True)` whose input raised kept its modified + time, so it and the nodes below it were not stale. +- A load-only run read a side-effect pod's stream outside a job (directly, through another + stream outside a job, or bound to a source slot), which delivered every row, although a + load-only run delivers nothing; it also read a function pod's stream outside a job, which + computed. +- An operator job node without a database did not record a load-only run as one. +- A standalone side-effect job node interrupted while committing its invocation log kept a + current run, so the next run neither ran nor committed. + +**Fix:** stamp before `on_node_end`; build the computed view inside the clearing block; +`RunSummary(ran_nothing=...)` from the run plan; clear the node when the load-only input +raises; the run plan and a source slot's `run()` refuse (`ValueError`, before reading) a +load-only run that would read a `SideEffectPodStream` or a `FunctionPodStream` +(`node_run.refuse_unloadable_inputs`, walking streams outside a job and bound source slots; +developer, November 14, 2026: "Refuse is consistent"); the no-database operator branch +records `load_only`; the side-effect node's log commit clears the node on any interruption. Two review comments did not +reproduce or are tracked elsewhere: a node run outside `job.run()` commits every row, so no +pre-run flush is needed; a function node with no database holding no rows after an async run is +ITL-1097. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 556ef5b56..cf691ea5d 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -577,6 +577,14 @@ incomplete: a group formed after it may lack members, and the result stored for group by an earlier run may then be loaded for it. `run_async(load_only=True)` does the same on the asynchronous orchestrator. +A load-only run reads only job nodes, sources and streams that neither compute nor deliver. +A pod's stream built outside the job (a function pod's, which computes when it is read, or a +side-effect pod's, which delivers) is refused with a `ValueError` before anything is read, +whether a node of the job reads it, a source slot is bound to it or an operator's stream +outside the job reads it. Build the pod inside the job, so its node loads its stored results, +or run without `load_only`. (Refusing is the consistent choice for now; ITL-1130 re-evaluates +it.) + ### Running again: what a run runs `job.run()`, `await job.run_async()` and `job.nodes[label].run()` decide what to run from @@ -832,14 +840,16 @@ notify_job.run() node = notify_job.nodes["notify"] print(node.as_table().num_rows, len(list(node.iter_data())), len(sent)) # 3 3 3 -node.run() # delivers only what the invocation log does not record: nothing here +node.run() # holds a current run: runs nothing and delivers nothing print(len(sent)) # 3 ``` - With `track_completion=True` (the default), a run delivers only the rows whose delivery the - node's invocation log does not record. With `track_completion=False`, every run, a second - `job.run()` or `node.run()`, delivers every row again; reads still deliver nothing. + node's invocation log does not record. With `track_completion=False`, a run delivers every + row. A node that holds a current run is not run again, so a second `job.run()` or + `node.run()` delivers nothing; call `clear_cache()` on the node first to deliver again. + Reads never deliver. - The rows a read returns are those the last run in this process passed on: under `on_error="log"`, a row whose delivery failed is left out with `drop_on_failure=True` (the default) and kept with `drop_on_failure=False`, as for the nodes downstream. diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 7c695088a..26e1ed8d4 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -302,7 +302,11 @@ def run_job_node( ``RunSummary`` or ``None``. caller: The frame of the ``run()`` caller. """ - from orcapod.pipeline.node_run import depends_on_job_nodes, run_with_dependencies + from orcapod.pipeline.node_run import ( + depends_on_job_nodes, + refuse_unloadable_inputs, + run_with_dependencies, + ) from orcapod.pipeline.run_summary import warn_row_failures if node.holds_current_run(load_only=load_only): @@ -310,6 +314,8 @@ def run_job_node( if depends_on_job_nodes(node): summary = run_with_dependencies(node, load_only=load_only) else: + if load_only: + refuse_unloadable_inputs(node) summary = run_alone() if summary is None: return @@ -1767,8 +1773,9 @@ def _execute_load_only( if entry[1] is not None: output.append((entry[0], entry[1])) self._set_run_rows(loaded_ids, load_only=True, not_stored=not_stored) - ctx_obs.on_node_end(node_label, node_hash) + # Current before the observer hears the end, so a read there sees the rows. self._update_modified_time() + ctx_obs.on_node_end(node_label, node_hash) except BaseException: # A run that raised (an observer at the node's end included) defines no rows. self.clear_cache() @@ -1822,8 +1829,9 @@ async def _async_execute_load_only( if entry[1] is not None: await output.send((entry[0], entry[1])) self._set_run_rows(loaded_ids, load_only=True, not_stored=not_stored) - ctx_obs.on_node_end(node_label, node_hash) + # Current before the observer hears the end, so a read there sees the rows. self._update_modified_time() + ctx_obs.on_node_end(node_label, node_hash) except BaseException: # A run that raised or was cancelled defines no rows. self.clear_cache() @@ -2345,6 +2353,10 @@ def execute( ) # The node's rows are those of this run's inputs, whatever else it caches. self._set_run_rows(emitted_entry_ids, failed=failed_rows) + # Mark this node as freshly computed before the observer hears the end, so + # a read there (and every later one) serves this run's rows instead of + # finding the node stale and clearing them (ITL-1121). + self._update_modified_time() ctx_obs.on_node_end(node_label, node_hash) except Exception as run_error: self._commit_after_failure(run_error) @@ -2355,9 +2367,6 @@ def execute( self.clear_cache() raise - # Mark this node as freshly computed so subsequent iter_data() calls - # skip the is_stale check and serve results directly from the in-memory cache. - self._update_modified_time() return output def _process_data_internal( @@ -4036,13 +4045,14 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: run_id=run_id, ) + # The run is complete: stamp the node as the sync execute() does, before + # the observer hears the end (a read there sees the rows, ITL-1121) and + # before the finally closes the output, so every downstream node, which + # stamps after it has seen that close, is newer than this one (ITL-1042). + self._update_modified_time() # Ended for the observer here, also when the call raises: not again below. node_open = False ctx_obs.on_node_end(node_label, node_hash) - # The run is complete: stamp the node as the sync execute() does, before - # the finally closes the output, so every downstream node, which stamps - # after it has seen that close, is newer than this one (ITL-1042). - self._update_modified_time() except asyncio.CancelledError as cancelled: # Another node's failure cancelled this one: commit what it wrote, as a # failing node does. The job's flush commits neither a node's ephemeral diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 51ff88427..d28fe12aa 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -1617,6 +1617,7 @@ def run(self, *, load_only: bool = False) -> StreamView: """ from orcapod.pipeline.node_run import ( depends_on_job_nodes, + refuse_unloadable_inputs, run_with_dependencies, ) from orcapod.pipeline.run_summary import warn_row_failures @@ -1633,6 +1634,8 @@ def run(self, *, load_only: bool = False) -> StreamView: return self._run_view(computed=False) if self._loaded: raise self._cannot_compute_error() + if load_only: + refuse_unloadable_inputs(self) if self.is_stale or not self._holds_run(load_only=load_only): # Stale, or a load-only run's output asked for by a full run: run again. self.clear_cache() @@ -1655,8 +1658,14 @@ def run(self, *, load_only: bool = False) -> StreamView: self._run_load_only = load_only else: self._cached_output_stream = self._compute_output() + # As in the branches with a database (ITL-1121). + self._run_load_only = load_only self._update_modified_time() computed = True + if computed: + # Building the view is part of the run: a failure there clears the + # node (ITL-1121). + return self._run_view(computed=True) except BaseException: # A run that raised defines no rows. self.clear_cache() diff --git a/src/orcapod/core/nodes/source_node.py b/src/orcapod/core/nodes/source_node.py index 6cc61b804..dde6cee50 100644 --- a/src/orcapod/core/nodes/source_node.py +++ b/src/orcapod/core/nodes/source_node.py @@ -594,15 +594,22 @@ def __init__( def run(self, *, load_only: bool = False) -> StreamView: """Read the bound source's rows and return a ``StreamView`` of all of them. - A source is read, never computed, so ``load_only`` changes nothing; a job - node's ``run()`` takes it (ITL-1095). + A source is read, never computed, so ``load_only`` changes nothing, except that + a load-only run refuses a bound stream that computes or delivers when read + (ITL-1121); a job node's ``run()`` takes it (ITL-1095). Args: load_only: Accepted for every job node; a source reads its rows either way. Raises: UnboundSourceError: When no concrete source is bound. + ValueError: If ``load_only`` and the bound stream computes or delivers + when read (``node_run.refuse_unloadable_inputs``). """ + if load_only: + from orcapod.pipeline.node_run import refuse_unloadable_inputs + + refuse_unloadable_inputs(self) return super().run() @property diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 80c32e95f..e77449503 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -1349,6 +1349,7 @@ def _finish_run( "/".join(self._name), run_node_summaries(prepared.exec_dag, counter, held=plan.held_counts), load_only=load_only, + ran_nothing=not plan.to_run, ) return self._last_run diff --git a/src/orcapod/pipeline/node_run.py b/src/orcapod/pipeline/node_run.py index 159578c94..2a0f18119 100644 --- a/src/orcapod/pipeline/node_run.py +++ b/src/orcapod/pipeline/node_run.py @@ -14,7 +14,7 @@ from __future__ import annotations -from collections.abc import Iterable, Sequence +from collections.abc import Iterable, Iterator, Sequence from dataclasses import dataclass from typing import TYPE_CHECKING, Any @@ -48,6 +48,84 @@ def depends_on_job_nodes(node: Any) -> bool: return any(_is_job_node(upstream) for upstream in node.upstreams) +def _streams_read_by(node: Any) -> Iterator[Any]: + """Yield the streams outside a job that a run of ``node`` reads. + + The node's inputs, the stream bound to a source slot among them, and, through every + stream outside a job, that stream's own inputs. A job node other than a source slot + is not entered: the run reads the rows the node holds (none if it has not run in + this process). Nor is an ``ArrowTableStream``: reading it reads only the table it + holds, whatever its ``upstreams`` name (ITL-1121). + + Args: + node: A job node; for a source slot, its bound stream is what it reads. + """ + from orcapod.core.nodes.source_node import SourceJobNode + from orcapod.core.streams.arrow_table_stream import ArrowTableStream + + pending: list[Any] = ( + [node.bound_source] if isinstance(node, SourceJobNode) else list(node.upstreams) + ) + seen: list[Any] = [] + while pending: + stream = pending.pop() + if stream is None or any(stream is other for other in seen): + continue + seen.append(stream) + if isinstance(stream, SourceJobNode): + pending.append(stream.bound_source) + continue + if _is_job_node(stream): + continue + yield stream + if not isinstance(stream, ArrowTableStream): + pending.extend(getattr(stream, "upstreams", ())) + + +def refuse_unloadable_inputs(node: Any) -> None: + """Refuse a load-only run of ``node`` that would read an input which computes or + delivers when it is read. + + A load-only run computes, delivers and writes nothing (ITL-1095), so it reads only + job nodes, sources and streams that neither compute nor deliver (developer, + November 14, 2026: "Refuse is consistent"; ITL-1121). Outside a job, a function + pod's stream (``FunctionPodStream``, a cached pod's included) computes its rows when + read, and a side-effect pod's stream (``SideEffectPodStream``) or node + (``SideEffectNode``, the blueprint a ``Pipeline`` records) delivers its side + effects. Such a stream is refused before anything is read, whether it is an input of + ``node``, bound to a source slot, or read through another stream outside a job (an + operator's stream over it). + + Args: + node: A job node a load-only run would run or read. + + Raises: + ValueError: If a run of ``node`` would read such a stream. + """ + from orcapod.core.function_pod import FunctionPodStream + from orcapod.side_effects import SideEffectNode, SideEffectPodStream + + for upstream in _streams_read_by(node): + # A SideEffectJobNode is a job node: _streams_read_by never yields one. + if isinstance(upstream, (SideEffectPodStream, SideEffectNode)): + what = ( + "a side-effect pod's stream outside a job, which delivers its side " + "effects when it is read, and a load-only run delivers nothing" + ) + elif isinstance(upstream, FunctionPodStream): + what = ( + "a function pod's stream outside a job, which computes when it is " + "read, and a load-only run computes nothing" + ) + else: + continue + raise ValueError( + f"A load-only run cannot run node {node.label!r}: its input " + f"{upstream.label!r} is {what}. Build the pod inside the job, or run " + "without load_only." + ) + + def _stores(nodes: list[Any]) -> list[ArrowDatabaseProtocol]: """The distinct stores the given nodes write to: pipeline and result databases.""" found: list[ArrowDatabaseProtocol] = [] @@ -114,7 +192,8 @@ def plan_run(graph: Any, *, load_only: bool, outside: Sequence[Any] = ()) -> Run is read only when a node that runs reads it; a stream outside the job (``outside``) is read for the nodes it feeds; every other node runs. The same plan serves ``job.run()`` (its whole graph) and ``node.run()`` (the node and what it - depends on). + depends on). A load-only plan refuses, before reading anything, a node to run whose + input computes or delivers when read (``refuse_unloadable_inputs``). Args: graph: The run's node graph (``topological_sort``, ``predecessors``). @@ -124,6 +203,9 @@ def plan_run(graph: Any, *, load_only: bool, outside: Sequence[Any] = ()) -> Run Returns: The plan. + + Raises: + ValueError: As ``refuse_unloadable_inputs``, for a load-only run. """ from orcapod.core.nodes.source_node import SourceJobNode from orcapod.core.nodes.staleness import RunHoldingNode @@ -154,6 +236,11 @@ def is_source(node: Any) -> bool: and not is_outside(node) and not any(node is c for c in current) ] + if load_only: + # Refused before any stream is read below: a load-only run cannot read an + # input that computes or delivers (ITL-1121). + for node in to_run: + refuse_unloadable_inputs(node) included: list[Any] = [] for node in order: feeds_a_run = any( @@ -222,6 +309,8 @@ def run_with_dependencies(target: Any, *, load_only: bool) -> RunSummary: Raises: StoreWriteError: If committing the run's writes failed. + ValueError: For a load-only run, if a node it would run reads an input that + computes or delivers when read (``refuse_unloadable_inputs``); nothing has run then. """ from orcapod.core.nodes.source_node import SourceJobNode from orcapod.pipeline.dag import OrcaDAG @@ -270,4 +359,5 @@ def visit(node: Any) -> None: run_node_summaries(graph, counter, held=plan.held_counts), load_only=load_only, target=target.label, + ran_nothing=not plan.to_run, ) diff --git a/src/orcapod/pipeline/run_summary.py b/src/orcapod/pipeline/run_summary.py index e3bc3dac2..e59e571f0 100644 --- a/src/orcapod/pipeline/run_summary.py +++ b/src/orcapod/pipeline/run_summary.py @@ -76,6 +76,9 @@ class RunSummary: it computed nothing; ``reused`` counts the rows it loaded. target: For the run of one node and the nodes it depends on (``node.run()``), that node's label; ``None`` for a job run. + ran_nothing: Whether the run ran no node: every node held a current run + (ITL-1095). Decided by the run's plan, which counts every kind of node, so a + run that ran only a side-effect or operator node is not "nothing to do". """ def __init__( @@ -85,6 +88,7 @@ def __init__( *, load_only: bool = False, target: str | None = None, + ran_nothing: bool = False, ) -> None: """Build the summary. @@ -93,11 +97,13 @@ def __init__( nodes: One entry per function node of the run, in run order. load_only: Whether the run loaded stored results only. target: The label of the node whose ``run()`` this was, if any. + ran_nothing: Whether the run ran no node (every node held a current run). """ self.job_name: str = job_name self.nodes: Mapping[str, NodeRunSummary] = {node.label: node for node in nodes} self.load_only: bool = load_only self.target: str | None = target + self.ran_nothing: bool = ran_nothing @property def failed_rows(self) -> int: @@ -152,10 +158,9 @@ def _headline(self) -> str: if self.target is None else f"node {self.target!r} and the nodes it depends on" ) - if self.nodes and all( - node.held and not (node.computed or node.reused or node.failed) - for node in self.nodes.values() - ): + # From what the run ran, not from the function nodes' counts: a run that ran + # only a side-effect or operator node did something (ITL-1121). + if self.ran_nothing: run = "Load-only run" if self.load_only else "Run" return f"{run} of {subject}: nothing to do; every node holds a current run." if self.load_only: diff --git a/src/orcapod/side_effects.py b/src/orcapod/side_effects.py index cc9e05b16..e8365cf0d 100644 --- a/src/orcapod/side_effects.py +++ b/src/orcapod/side_effects.py @@ -1163,7 +1163,12 @@ def execute( """ self._held_rows = None if load_only: - rows = list(input_stream.iter_data()) + try: + rows = list(input_stream.iter_data()) + except BaseException: + # A run that raised defines no rows, as on the async path (ITL-1121). + self.clear_cache() + raise self._hold(rows, load_only=True) return rows try: @@ -1350,6 +1355,11 @@ def _run_alone(self, *, load_only: bool) -> None: except Exception as exc: self.clear_cache() raise StoreWriteError(database.base_path, flush=True) from exc + except BaseException: + # An interruption leaves the commit's outcome unknown: the node holds no + # run, so the next run runs and commits again (ITL-1121). + self.clear_cache() + raise # ------------------------------------------------------------------ # Async execution diff --git a/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md b/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md new file mode 100644 index 000000000..08b6e892d --- /dev/null +++ b/superpowers/plans/2026-11-14-itl-1121-node-view-review-fixes.md @@ -0,0 +1,179 @@ +# ITL-1121: job node view review fixes, implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan +> task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Fix the defects a full CodeRabbit review of PR #381 found in PR #380's code (the job +node current-run view, ITL-1095), in a PR stacked on #380, without new node-view behaviour and +without any hash, golden or stored-table change. + +**Architecture:** Each fix goes where the defect lives: the function node's run paths (stamp +before `on_node_end`), the operator node's `run()` (lazy materialization inside the clearing +`try`), `RunSummary` (headline from what the run plan ran), the side-effect node's sync +load-only branch (clear on failure), and one load-only guard in `pipeline/node_run.py` that +refuses an input which delivers when read. + +**Tech Stack:** Python 3.11+, pytest, `uv run`. + +Branch: `eywalker/itl-1121-job-node-view-itl-1095-defects-found-by-a`, based on PR #380 head +`7de3512b`. Draft PR against `main`, top line "Stacked on #380: merge after it. Fixes ITL-1121". + +Reproduction scripts: `superpowers/scripts/itl_1121/c1..c6_*.py` (committed). + +## Coordinator decisions (November 14, 2026) + +| # | Reproduced | Decision | +|---|---|---| +| 1 | yes, sync / async / load-only sync / load-only async | fix; observer test on all four paths + the node holds its rows after the run | +| 2 | yes (no DB, `CacheMode.OFF`) | fix | +| 3 | no (node.run commits per row; evidence c3) | no change; reply with evidence | +| 4 | yes, side-effect AND operator node cleared below a held function node | fix: `RunSummary(..., ran_nothing=...)`; tests: side-effect cleared, operator cleared, nothing ran, job with no function node; sync, async, node.run() | +| 5 | yes, through `execute()` only | fix (as async) | +| 6 | yes; general rule for outside streams | (a): a load-only run refuses, before reading, an input that would deliver (`SideEffectPodStream`), all three node kinds, direct input and cascade. A computing outside stream (FunctionPodStream) stays: write its reproduction + 3 lines for the coordinator | +| 7 | ITL-1097 | no fix; point to it | +| 8 | yes | docs: held node delivers nothing; `clear_cache()` first | +| 9 | - | recording function in `test-objective/integration/test_caching_flows.py` | + +## Files + +- Modify `src/orcapod/core/nodes/function_node.py`: `execute` (~2300-2322), `_execute_load_only` + (~1731-1733), `_async_execute_load_only` (~1786-1788), `async_execute` (~4003-4007); + `run_job_node` (~306-312, guard 6). +- Modify `src/orcapod/core/nodes/operator_node.py`: `run` (~1540-1580, fix 2 and guard 6). +- Modify `src/orcapod/pipeline/run_summary.py`: `RunSummary.__init__`, `_headline`. +- Modify `src/orcapod/pipeline/job.py`: `_finish_run` (`RunSummary(...)`). +- Modify `src/orcapod/pipeline/node_run.py`: `run_with_dependencies` (`RunSummary(...)`, guard 6), + new `refuse_delivering_inputs(node)`. +- Modify `src/orcapod/side_effects.py`: `SideEffectJobNode.execute` load-only branch; `run` uses + `run_job_node` (guard covered there). +- Modify `docs/concepts/function-pods.md` (~836-842). +- Modify `test-objective/integration/test_caching_flows.py` (`test_second_run_uses_cache`). +- Create `tests/test_pipeline/test_node_view_review_fixes.py` (all regression tests). +- Create `changelog.d/ITL-1121.md` (### Fixed), `design_issues/ITL-1121-NV1.md`. + +## Task 1: stamp before `on_node_end` (comment 1) + +- [ ] Test: observer subclassing `NoOpObserver` (contextualize returns self) reads + `job.nodes[label].as_table().num_rows` in `on_node_end`; parametrize orchestrator + sync/async and load_only False/True (load_only: a first job over the same store stores the + results). Assert read == 3 in the hook, and after the run `as_table().num_rows == 3` and + `holds_current_run(load_only=load_only)`. +- [ ] Run: fails (0 rows). +- [ ] Fix: in each of the four paths, call `self._update_modified_time()` right after + `self._set_run_rows(...)` and before `ctx_obs.on_node_end(...)`; delete the later stamp. A + raising observer still clears the node (the except branches call `clear_cache()`). +- [ ] Run the test and `tests/test_pipeline/test_node_staleness_after_run.py`, + `test_job_node_current_run_view.py`, `test_observer.py`: pass. Commit `fix(function_node): ...`. + +## Task 2: operator lazy materialization inside the clearing try (comment 2) + +- [ ] Test: `OperatorJobNode(MapData({"y": "z"}), (flaky.pod(base),), pipeline_database=db, + cache_mode=CacheMode.OFF)` for `db` None and `InMemoryArrowDatabase()`; the pod raises: + `run()` raises; then `holds_current_run(load_only=False) is False`, `is_stale`, and + `as_table().num_rows == 0`; after the pod recovers `run()` gives 2 rows. +- [ ] Fix: in `OperatorJobNode.run`, build the view inside the `try` for the branch that + computed (`return self._run_view(computed=computed)` moved into the `try` when + `computed`), keeping `except BaseException: self.clear_cache(); raise`. +- [ ] Run test + `tests/test_core/operators/test_operator_node*.py`. Commit. + +## Task 3: summary headline from what ran (comment 4) + +- [ ] Tests (sync, async; and `node.run()`): + - function -> tap; run; `tap.clear_cache()`; run: delivered 3 and the headline is not + "nothing to do" (`"Run of job 'j': all rows succeeded."`). + - function -> select (operator); `op.clear_cache()`; run: headline not "nothing to do". + - nothing ran: second run of an unchanged job says "nothing to do". + - a job with no function node (source -> select): a second run says "nothing to do"; the + first says "all rows succeeded". +- [ ] Fix: `RunSummary.__init__(..., ran_nothing: bool = False)` stored as `self.ran_nothing` + (documented in Attributes and Args); `_headline` returns the "nothing to do" line iff + `self.ran_nothing`. Callers: `job._finish_run` and `node_run.run_with_dependencies` pass + `ran_nothing=not plan.to_run`. +- [ ] Run test + `tests/test_pipeline/test_job_run_state_rule.py test_job_run_summary.py + test_node_run_cascade.py`. Commit. + +## Task 4: sync load-only execute clears on failure (comment 5) + +- [ ] Test: job `rowwise -> tap -> down`; `job.run()`; `tap.execute(failing, load_only=True)` + raises; `tap.is_stale` and `down.is_stale`, both read 0 rows. (`execute` is the node's + protocol entry point; this test is about it: one-line comment.) +- [ ] Fix: wrap the load-only branch in `try: ... except BaseException: self.clear_cache(); raise`. +- [ ] Commit. + +## Task 5: a load-only run refuses an input that delivers (comment 6) + +- [ ] Tests: standalone `SideEffectJobNode`, `FunctionJobNode`, `OperatorJobNode` over a + `SideEffectPodStream`: `run(load_only=True)` raises `ValueError` and nothing is delivered; + cascade: a job node over a standalone node over a `SideEffectPodStream`... (built as nodes + outside a job, `FunctionJobNode(pod, SideEffectJobNode(...))`, only if constructible; else + the direct cases plus the cascade through `run_with_dependencies` with a job node input). + Full runs are unchanged (deliver as before). +- [ ] Fix: `node_run.refuse_delivering_inputs(node)` raises `ValueError` naming the node and + the input when an upstream is a `SideEffectPodStream`. Called when `load_only` in + `run_job_node` (alone path, before `run_alone`), in `OperatorJobNode.run` (alone path), and in + `run_with_dependencies` for every node of `plan.to_run` before the orchestrator starts. +- [ ] Write c7 reproduction for FunctionPodStream computing in a load-only run (already in c6) + and three lines for the coordinator. +- [ ] Commit. + +## Task 6: docs (8), test-objective (9), changelog, design issue + +- [ ] `function-pods.md`: the sample comment `node.run() # holds a current run: delivers + nothing`; the bullet: with `track_completion=False`, a run delivers every row, but a node + that holds a current run is not run again: `clear_cache()` it first. Mention the load-only + refusal sentence in the load-only paragraph if one exists. +- [ ] `test_caching_flows.py`: recording `_double`, assert no call during `node2.run()`. +- [ ] `changelog.d/ITL-1121.md` (### Fixed), `design_issues/ITL-1121-NV1.md` (resolved). +- [ ] `uv run python scripts/assemble_changelog.py --check`, `scripts/design_issues.py --check`, + docs tests. Commit. + +## Task 7: finish + +- [ ] ruff format/check, basedpyright exit code, full `uv run pytest tests/` and + `test-objective/`. Push, draft PR, `orca linear attach`, `orca worktree set --pr`. +- [ ] Reply on the nine CodeRabbit threads/review-body findings on #381 (signed). +- [ ] CI green, CodeRabbit round on the new PR addressed; worker_done. + +## Progress notes + +- 2026-11-14: reproduction table sent and approved (see decisions). Scripts committed (86902ec0). +- 2026-11-14 (session stopped at usage limit): DONE and pushed: fixes 1, 2, 4, 5, 6 (each its own + commit, tests in tests/test_pipeline/test_node_view_review_fixes.py, 32 pass), docs 8, test 9. + Fix 6 lives in node_run.plan_run (refuse_delivering_inputs) because plan_run reads outside + streams as held rows; plus run_job_node / OperatorJobNode.run alone paths. + Observation for the coordinator: a node.run() of a node holding a current run makes no summary; + node.last_run stays the previous run's (existing #380 behaviour, not changed). + LEFT: changelog.d/ITL-1121.md (### Fixed), design_issues/ITL-1121-NV1.md, the FunctionPodStream + three lines for the coordinator (repro in superpowers/scripts/itl_1121/c6_*.py), basedpyright, + full `uv run pytest tests/` + test-objective, draft PR (top line "Stacked on #380: merge after + it. Fixes ITL-1121"), orca linear attach / worktree set --pr, signed replies on the nine #381 + findings (3: evidence c3; 7: ITL-1097), CI green, CodeRabbit round, worker_done. +- 2026-11-14 (resumed): changelog, design issue, draft PR #388, #381 replies (6 threads + 1 top-level), + CodeRabbit round 1 on #388 fixed (C, D/F, H, I; 75324abc) and answered (A, B, E, G, J, K, L), + KeyboardInterrupt log-commit fix (c9). Developer decision: a load-only run also refuses a + computing outside stream (d06ffcf7, node_run.refuse_unloadable_inputs). main merged (#382, #383). + LEFT: name the coordinator's follow-up issue in the reply to finding G (thread 4574191943); + CI and a CodeRabbit review of the head; worker_done. + +## Merge of main after #380 merged (task_f017ca8a173c) + +#380 merged as e4b4319c (it had merged #375 ITL-912 typed zero-row tables and #387 ITL-1111 +schema memo). Steps: +- [x] `git merge origin/main` (no rebase); resolve function_node.py, operator_node.py, + side_effects.py, node_run.py, job.py, CLAUDE.md, .zed/rules keeping every #388 fix and main's + #375 / #387 behaviour. +- [x] Doc fix: CLAUDE.md / .zed/rules "A failed end-of-run commit clears every node of the run" + -> only the nodes that ran (`run_with_dependencies` -> `commit_run(stores, plan.to_run)`). +- [x] Re-run every script in superpowers/scripts/itl_1121/: each shows the fixed behaviour. +- [x] Tests: tests/test_pipeline, tests/test_core/{nodes,operators,function_pod,side_effect_pod}, + golden (3.12; async files also 3.11; one process). +- [ ] One push; CI green; one signed `@coderabbitai review`; plan to coordinator before code. +- 2026-11-14 (merge task): merged #380's final head 8c44199b first (1aa7ca2f, no conflicts: + the true base), then origin/main e4b4319c (6b5db050; main's tree equals 8c44199b, so every + conflict of the squash took this branch's side). Doc fix 8455c2f0. All itl_1121 scripts show + the fixed behaviour. Tests: pipeline/nodes/operators/function_pod/side_effect_pod/golden on + 3.12 6434 passed 11 skipped 9 xfailed; async files + review-fix tests on 3.11 651 passed. + LEFT: CI, one CodeRabbit review, plan to coordinator, worker_done. +- 2026-11-14: CodeRabbit 4575098786 (5dd8d6a3 round): #380 behaviour, also on main; declined with repro c11 (coordinator files it); _streams_read_by docstring corrected. CR review at 21:07Z rate limited; retries ~21:28Z, ~21:48Z. +- 2026-11-14: merged #390 (653c138e; one conflict in OperatorJobNode.run, kept _compute_output + _run_load_only) and #389 (810d1b03, clean). Scripts unchanged. 3.12 targets 9348 passed; async 3.11 651 passed. CR retry 1 at 21:38Z rate limited; last retry ~21:58Z. +- 2026-11-14: CodeRabbit round 2 (review 5907415238 of 9fc663d7): A 4575874462 (ArrowTableStream upstreams over-refused) and B 4575874471 (blueprint SideEffectNode delivered in a load-only run) reproduced (c12), approved, fixed in one commit with 13 tests. 3.12 targets 9438 passed; 3.11 async 664 passed. diff --git a/superpowers/scripts/itl_1121/c10_held_node_run_keeps_last_run.py b/superpowers/scripts/itl_1121/c10_held_node_run_keeps_last_run.py new file mode 100644 index 000000000..b34049405 --- /dev/null +++ b/superpowers/scripts/itl_1121/c10_held_node_run_keeps_last_run.py @@ -0,0 +1,25 @@ +"""Observation (not changed): node.run() of a node that holds a current run returns at once +and makes no summary, so node.last_run stays the summary of the run before it.""" + +from orcapod import PipelineJob, function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.sources import DictSource + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + return x + 1 + + +job = PipelineJob(name="j", database=InMemoryArrowDatabase()) +with job: + inc.pod( + DictSource([{"k": 1, "x": 1}], tag_columns=["k"], source_id="s"), + label="rowwise", + ) +node = job.nodes["rowwise"] +node.run() +first = node.last_run +node.run() # held: runs nothing +print(f"second run() left last_run unchanged: {node.last_run is first}") +print(repr(node.last_run).splitlines()[0]) diff --git a/superpowers/scripts/itl_1121/c11_job_node_behind_outside_stream.py b/superpowers/scripts/itl_1121/c11_job_node_behind_outside_stream.py new file mode 100644 index 000000000..f7e78116e --- /dev/null +++ b/superpowers/scripts/itl_1121/c11_job_node_behind_outside_stream.py @@ -0,0 +1,59 @@ +"""CodeRabbit 4575098786 (out of scope, #380 behaviour, also on main e4b4319c): node.run() +does not schedule a job node reached through a stream outside a job; the run reads 0 rows.""" + +from typing import Any + +import pyarrow as pa +from orcapod import function_pod +from orcapod.core.nodes.function_node import FunctionJobNode +from orcapod.core.nodes.operator_node import OperatorJobNode +from orcapod.core.operators import MapData +from orcapod.core.streams import ArrowTableStream +from orcapod.databases import InMemoryArrowDatabase + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + return x + 1 + + +@function_pod(output_keys="w") +def dbl(z: int) -> int: + return 2 * z + + +inc_pod: Any = inc.pod +dbl_pod: Any = dbl.pod + +schema = pa.schema( + [ + pa.field("id", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] +) +base = ArrowTableStream( + pa.table({"id": [1, 2, 3], "x": [1, 2, 3]}, schema=schema), tag_columns=["id"] +) +for via in ["function", "operator"]: + first = FunctionJobNode(inc_pod, base, pipeline_database=InMemoryArrowDatabase()) + mid = MapData({"y": "z"})( + first + ) # an operator's stream outside a job over a job node + if via == "function": + second = FunctionJobNode( + dbl_pod, mid, pipeline_database=InMemoryArrowDatabase() + ) + else: + second = OperatorJobNode( + operator=MapData({"z": "q"}), + input_streams=(mid,), + pipeline_database=InMemoryArrowDatabase(), + ) + print(type(mid).__name__, "first ran?", first.holds_current_run(load_only=False)) + try: + view = second.run() + print( + f"{via}: second.run() rows={view.as_table().num_rows}; first holds a run: {first.holds_current_run(load_only=False)}; summary: {second.last_run}" + ) + except Exception as exc: + print(f"{via}: raised {type(exc).__name__}: {exc}") diff --git a/superpowers/scripts/itl_1121/c12_cr388_round2_traversal.py b/superpowers/scripts/itl_1121/c12_cr388_round2_traversal.py new file mode 100644 index 000000000..a24dace2a --- /dev/null +++ b/superpowers/scripts/itl_1121/c12_cr388_round2_traversal.py @@ -0,0 +1,101 @@ +"""PR #388 CodeRabbit round 2 (review of 9fc663d7): the load-only input traversal. + +A (4575874462): an ArrowTableStream holds its table and never reads its upstreams, yet a + load-only run over one built with upstreams=(a function pod's stream,) is refused. +B (4575874471): a blueprint SideEffectNode outside a job delivers when it is read; a + load-only run over it (a node's input, or bound to a source slot) is not refused. +""" + +from typing import Any + +import pyarrow as pa + +from orcapod import PipelineJob, function_pod +from orcapod.core.nodes.function_node import FunctionJobNode +from orcapod.core.streams import ArrowTableStream +from orcapod.databases import InMemoryArrowDatabase +from orcapod.side_effects import SideEffectNode, tap_pod +from orcapod.sources import DictSource + +UP: list[int] = [] +CALLS: list[int] = [] + + +def deliver_up(x: int, ctx: Any) -> None: + UP.append(x) + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + CALLS.append(x) + return x + 1 + + +@function_pod(output_keys="w") +def twice(x: int) -> int: + return 2 * x + + +inc_pod: Any = inc.pod +twice_pod: Any = twice.pod +schema = pa.schema( + [ + pa.field("k", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] +) + + +def base() -> Any: + return ArrowTableStream( + pa.table({"k": [1, 2], "x": [1, 2]}, schema=schema), tag_columns=["k"] + ) + + +def blueprint_tap() -> Any: + tap: Any = tap_pod(deliver_up) + return SideEffectNode(tap, base()) + + +# A +held = ArrowTableStream( + pa.table({"k": [1, 2], "x": [1, 2]}, schema=schema), + tag_columns=["k"], + upstreams=(inc_pod(base()),), +) +node = FunctionJobNode(twice_pod, held, pipeline_database=InMemoryArrowDatabase()) +try: + view = node.run(load_only=True) + print( + f"A: run(load_only=True) accepted: status {view.status}; function calls {len(CALLS)}" + ) +except ValueError as exc: + print(f"A: refused: {exc}; function calls {len(CALLS)}") + +# B, as a node's input +UP.clear() +node = FunctionJobNode( + twice_pod, blueprint_tap(), pipeline_database=InMemoryArrowDatabase() +) +try: + node.run(load_only=True) + print(f"B input: run(load_only=True) ran: delivered {len(UP)}") +except ValueError as exc: + print(f"B input: refused: {exc}; delivered {len(UP)}") + +# B, bound to a source slot +db = InMemoryArrowDatabase() +job = PipelineJob(name="j", database=db) +with job: + src = DictSource( + [{"k": 1, "x": 1}, {"k": 2, "x": 2}], tag_columns=["k"], source_id="s" + ) + twice.pod(src, label="tw") +job.run() +UP.clear() +try: + job.bind(sources={"s": blueprint_tap()}) + job.run(load_only=True) + print(f"B slot: job.run(load_only=True) ran: delivered {len(UP)}") +except ValueError as exc: + print(f"B slot: refused: {exc}; delivered {len(UP)}") diff --git a/superpowers/scripts/itl_1121/c1_observer_reads_in_on_node_end.py b/superpowers/scripts/itl_1121/c1_observer_reads_in_on_node_end.py new file mode 100644 index 000000000..0f4406a0d --- /dev/null +++ b/superpowers/scripts/itl_1121/c1_observer_reads_in_on_node_end.py @@ -0,0 +1,59 @@ +"""Comment 1: a function job node is stamped after on_node_end, so an observer that reads +the node in on_node_end finds it stale and clears it.""" + +from typing import Any + +from orcapod import PipelineJob, function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.pipeline.observer import NoOpObserver +from orcapod.sources import DictSource + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + return x + 1 + + +class Reader(NoOpObserver): + def __init__(self, job: Any) -> None: + self.job = job + self.seen: dict[str, int] = {} + + def contextualize(self, *identity_path: str) -> "Reader": + return self + + def on_node_end(self, node_label: str, node_hash: str) -> None: + node = self.job.nodes.get(node_label) + if node is not None: + self.seen[node_label] = node.as_table().num_rows + + +def build(db: Any) -> PipelineJob: + job = PipelineJob(name="j", database=db) + with job: + inc.pod( + DictSource( + [{"k": i, "x": i} for i in range(3)], tag_columns=["k"], source_id="s" + ), + label="rowwise", + ) + return job + + +for name, orch in [ + ("sync", SyncPipelineOrchestrator), + ("async", AsyncPipelineOrchestrator), +]: + for load_only in (False, True): + db = InMemoryArrowDatabase() + if load_only: + build(db).run() # store the results first + job = build(db) + obs = Reader(job) + job.run(orchestrator=orch(), observer=obs, load_only=load_only) + node = job.nodes["rowwise"] + print( + f"{name:5} load_only={load_only!s:5}: read in on_node_end={obs.seen.get('rowwise')}, " + f"after run: rows={node.as_table().num_rows} holds_current_run={node.holds_current_run(load_only=load_only)}" + ) diff --git a/superpowers/scripts/itl_1121/c2_operator_lazy_failure.py b/superpowers/scripts/itl_1121/c2_operator_lazy_failure.py new file mode 100644 index 000000000..1a4c78100 --- /dev/null +++ b/superpowers/scripts/itl_1121/c2_operator_lazy_failure.py @@ -0,0 +1,61 @@ +"""Comment 2: an operator job node's run() over a lazy output (no database, or +CacheMode.OFF) materializes it in _run_view(), outside the try that clears the node.""" + +import pyarrow as pa + +from orcapod import function_pod +from orcapod.core.nodes.operator_node import OperatorJobNode +from orcapod.core.operators import MapData +from orcapod.core.streams import ArrowTableStream +from orcapod.databases import InMemoryArrowDatabase +from orcapod.types import CacheMode + +FAIL = {"on": True} + + +@function_pod(output_keys="y") +def flaky(x: int) -> int: + if FAIL["on"]: + raise RuntimeError("boom") + return x + 1 + + +schema = pa.schema( + [ + pa.field("id", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] +) +base = ArrowTableStream( + pa.table({"id": [1, 2], "x": [1, 2]}, schema=schema), tag_columns=["id"] +) + +for label, db in [("no database", None), ("CacheMode.OFF", InMemoryArrowDatabase())]: + FAIL["on"] = True + upstream = flaky.pod(base) # a pod stream outside a job: lazy, raises when read + node = OperatorJobNode( + operator=MapData({"y": "z"}), + input_streams=(upstream,), + pipeline_database=db, + cache_mode=CacheMode.OFF, + ) + try: + node.run() + except Exception as exc: + print(f"{label}: run() raised {type(exc).__name__}: {exc}") + print( + f" after the failed run: holds_current_run={node.holds_current_run(load_only=False)} is_stale={node.is_stale} " + f"kept output installed={node._cached_output_stream is not None}" + ) # private: what the comment is about + try: + print(f" as_table() rows={node.as_table().num_rows}") + except Exception as exc: + print(f" as_table() raised {type(exc).__name__}: {exc}") + print(f" head(): {node.head().status}") + FAIL["on"] = False + try: + print( + f" second run() after the input recovers: rows={node.run().as_table().num_rows}" + ) + except Exception as exc: + print(f" second run() raised {type(exc).__name__}: {exc}") diff --git a/superpowers/scripts/itl_1121/c3_node_run_pending_writes.py b/superpowers/scripts/itl_1121/c3_node_run_pending_writes.py new file mode 100644 index 000000000..d6cb7fabc --- /dev/null +++ b/superpowers/scripts/itl_1121/c3_node_run_pending_writes.py @@ -0,0 +1,102 @@ +"""Comment 3: node.run() starts without the pre-run flush that job.run() makes (ITL-919). + +The hazard ITL-919 guards against: a table-committing store reaches batch_size on a +table mid-run and commits pipeline records whose results are still pending. Check, on a +Delta store with a small batch_size and rows left pending before the run, that every +committed pipeline-record version only references committed results after node.run(). +""" + +import tempfile +from pathlib import Path + +from typing import Any + +import deltalake +import pyarrow as pa + +from orcapod import PipelineJob, function_pod +from orcapod.databases import DeltaTableDatabase +from orcapod.databases.storage_utils import delta_to_pyarrow_dataset +from orcapod.sources import DictSource + + +@function_pod(output_keys="y") +def double(x: int) -> int: + return 2 * x + + +def col(table: deltalake.DeltaTable, name: str) -> set[object]: + return set( + delta_to_pyarrow_dataset(table).to_table(columns=[name])[name].to_pylist() + ) + + +# Every Delta commit, in the order it happened: (table directory, rows written). Commit +# timestamps have millisecond resolution, so equal timestamps could not order two commits; +# the order of the write calls does. +COMMITS: list[tuple[str, pa.Table]] = [] +_real_write = deltalake.write_deltalake + + +def _recording_write(table_or_uri: Any, data: Any, *args: Any, **kwargs: Any) -> Any: + COMMITS.append( + (str(table_or_uri), pa.table(data) if not isinstance(data, pa.Table) else data) + ) + return _real_write(table_or_uri, data, *args, **kwargs) + + +deltalake.write_deltalake = _recording_write # pyright: ignore[reportAttributeAccessIssue] # a recording wrapper for this script only + + +def check(root: Path) -> None: + pipeline = [ + t for path, t in COMMITS if "_result" not in path and "leftover" not in path + ] + results = [t for path, t in COMMITS if "_result" in path] + tables = { + ("result" if "_result" in t.relative_to(root).parts else "pipeline"): t + for t in {log.parent for log in root.rglob("_delta_log")} + if t.name != "leftover" + } + history = { + kind: len(deltalake.DeltaTable(str(t)).history()) for kind, t in tables.items() + } + print( + f" recorded commits: pipeline {len(pipeline)}, result {len(results)}; " + f"table histories: {history}" + ) + committed: set[object] = set() + bad = 0 + for path, table in COMMITS: + if "_result" in path: + committed |= set(table.column("__record_id").to_pylist()) + elif "leftover" not in path: + if not set(table.column("__data_id").to_pylist()) <= committed: + bad += 1 + print(f" pipeline commits referencing a result not committed before them: {bad}") + + +for batch_size, n in [(50, 120), (5, 40)]: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + store = DeltaTableDatabase(root, batch_size=batch_size) + for i in range(batch_size - 1): # pending, never flushed + store.add_record( + ("leftover",), + f"left-{i}".encode(), + pa.table({"v": pa.array([i], pa.int64())}), + ) + job = PipelineJob(name="j", database=store) + with job: + double.pod( + DictSource( + [{"k": i, "x": i} for i in range(n)], + tag_columns=["k"], + source_id="s", + ), + label="dbl", + ) + COMMITS.clear() + print(f"batch_size={batch_size}, rows={n}: node.run()") + job.nodes["dbl"].run() + check(root) diff --git a/superpowers/scripts/itl_1121/c4_summary_side_effect.py b/superpowers/scripts/itl_1121/c4_summary_side_effect.py new file mode 100644 index 000000000..8c40fe8f4 --- /dev/null +++ b/superpowers/scripts/itl_1121/c4_summary_side_effect.py @@ -0,0 +1,78 @@ +"""Comment 4: function -> side-effect job; run, clear only the side-effect node, run again.""" + +from typing import Any + +from orcapod import PipelineJob, function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.side_effects import tap_pod +from orcapod.sources import DictSource + +DELIVERED: list[int] = [] + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + return x + 1 + + +def deliver(y: int, ctx: Any) -> None: + DELIVERED.append(y) + + +for name, orch in [ + ("sync", SyncPipelineOrchestrator), + ("async", AsyncPipelineOrchestrator), +]: + for via in ("job.run", "node.run"): + DELIVERED.clear() + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + rows = inc.pod( + DictSource( + [{"k": i, "x": i} for i in range(3)], + tag_columns=["k"], + source_id="s", + ), + label="rowwise", + ) + tap: Any = tap_pod(deliver) + tap.process(rows, label="tap") + job.run(orchestrator=orch()) + first = len(DELIVERED) + job.nodes["tap"].clear_cache() + DELIVERED.clear() + if via == "job.run": + summary = job.run(orchestrator=orch()) + else: + if name == "async": + continue # node.run() is sync only + job.nodes["tap"].run() + summary = job.nodes["tap"].last_run + print( + f"--- {name} {via}: first run delivered {first}; second run delivered {len(DELIVERED)}" + ) + print(repr(summary)) + +# The same with an operator node below the held function node instead of a side effect. +for name, orch in [ + ("sync", SyncPipelineOrchestrator), + ("async", AsyncPipelineOrchestrator), +]: + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + rows = inc.pod( + DictSource( + [{"k": i, "x": i} for i in range(3)], tag_columns=["k"], source_id="s" + ), + label="rowwise", + ) + rows_any: Any = rows + rows_any.select_data_columns(["y"], label="op") + job.run(orchestrator=orch()) + job.nodes["op"].clear_cache() + summary = job.run(orchestrator=orch()) + print( + f"--- {name} job.run, operator node cleared: op holds_current_run={job.nodes['op'].holds_current_run(load_only=False)}" + ) + print(repr(summary)) diff --git a/superpowers/scripts/itl_1121/c5_load_only_execute_failure.py b/superpowers/scripts/itl_1121/c5_load_only_execute_failure.py new file mode 100644 index 000000000..f9924d46f --- /dev/null +++ b/superpowers/scripts/itl_1121/c5_load_only_execute_failure.py @@ -0,0 +1,84 @@ +"""Comment 5: a sync load-only execute() whose input raises does not clear the node +(the async load-only branch and the full-run branch do).""" + +import asyncio +from typing import Any + +import pyarrow as pa + +from orcapod import PipelineJob, function_pod +from orcapod.channels import Channel +from orcapod.core.streams import ArrowTableStream +from orcapod.databases import InMemoryArrowDatabase +from orcapod.side_effects import tap_pod +from orcapod.sources import DictSource + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + return x + 1 + + +@function_pod(output_keys="z") +def boom(y: int) -> int: + raise RuntimeError("input failed") + + +def deliver(y: int, ctx: Any) -> None: + pass + + +def build() -> PipelineJob: + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + rows = inc.pod( + DictSource( + [{"k": i, "x": i} for i in range(3)], tag_columns=["k"], source_id="s" + ), + label="rowwise", + ) + tap: Any = tap_pod(deliver) + tapped: Any = tap.process(rows, label="tap") + tapped.select_data_columns(["y"], label="down") + return job + + +schema = pa.schema( + [ + pa.field("k", pa.int64(), nullable=False), + pa.field("y", pa.int64(), nullable=False), + ] +) +failing = boom.pod( + ArrowTableStream(pa.table({"k": [1], "y": [1]}, schema=schema), tag_columns=["k"]) +) + + +class FailingChannel: + def __aiter__(self): + return self + + async def __anext__(self): + raise RuntimeError("input failed") + + +for mode in ("sync full", "sync load_only", "async load_only"): + job = build() + job.run() + tap, down = job.nodes["tap"], job.nodes["down"] + try: + if mode == "sync full": + tap.execute(failing) + elif mode == "sync load_only": + tap.execute(failing, load_only=True) + else: + out = Channel() + asyncio.run( + tap.async_execute([FailingChannel()], out.writer, load_only=True) + ) + except Exception as exc: + print(f"{mode}: raised {type(exc).__name__}: {exc}") + print( + f" tap: is_stale={tap.is_stale} rows={tap.as_table().num_rows}; " + f"down (ran before): is_stale={down.is_stale} rows={down.as_table().num_rows}" + ) diff --git a/superpowers/scripts/itl_1121/c6_load_only_over_side_effect_stream.py b/superpowers/scripts/itl_1121/c6_load_only_over_side_effect_stream.py new file mode 100644 index 000000000..f376e30f1 --- /dev/null +++ b/superpowers/scripts/itl_1121/c6_load_only_over_side_effect_stream.py @@ -0,0 +1,79 @@ +"""Comment 6: a SideEffectJobNode over a standalone SideEffectPodStream, run with +load_only=True, delivers the upstream stream's side effects while reading it. + +Compared with the general case: a FunctionJobNode over a FunctionPodStream (a stream +outside a job), run with load_only=True, computes the upstream function while reading it. +""" + +from typing import Any + +import pyarrow as pa + +from orcapod import function_pod +from orcapod.core.nodes.function_node import FunctionJobNode +from orcapod.core.streams import ArrowTableStream +from orcapod.databases import InMemoryArrowDatabase +from orcapod.side_effects import SideEffectJobNode, tap_pod + +UP: list[int] = [] +OWN: list[int] = [] +CALLS: list[int] = [] + + +def deliver_up(x: int, ctx: Any) -> None: + UP.append(x) + + +def deliver_own(x: int, ctx: Any) -> None: + OWN.append(x) + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + CALLS.append(x) + return x + 1 + + +@function_pod(output_keys="z") +def dbl(y: int) -> int: + return 2 * y + + +schema = pa.schema( + [ + pa.field("k", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] +) +base = ArrowTableStream( + pa.table({"k": [1, 2, 3], "x": [1, 2, 3]}, schema=schema), tag_columns=["k"] +) + +tap_up: Any = tap_pod(deliver_up) +upstream: Any = tap_up.process(base) # a SideEffectPodStream, outside a job +print("input type:", type(upstream).__name__) +tap_own: Any = tap_pod(deliver_own) +node = SideEffectJobNode(tap_own, upstream) +# Before ITL-1121 this delivered every row of the upstream stream (3, then 6 after +# clear_cache()); since then the load-only run refuses the input before reading it. +try: + view = node.run(load_only=True) + print(f"side-effect node, run(load_only=True): rows {view.as_table().num_rows}") +except ValueError as exc: + print(f"side-effect node, run(load_only=True) refused: {exc}") +print(f" upstream delivered {len(UP)}, own delivered {len(OWN)}") + +# A function pod's stream outside a job computes when read. Before the developer's decision +# (November 14, 2026, "Refuse is consistent") a load-only run read it and called the +# function; since then it is refused as well. +fn_upstream: Any = inc.pod(base) # a FunctionPodStream, outside a job +dbl_pod: Any = dbl.pod +fnode = FunctionJobNode(dbl_pod, fn_upstream, pipeline_database=InMemoryArrowDatabase()) +try: + fnode.run(load_only=True) + print( + f"function node over a pod stream, run(load_only=True): upstream calls {len(CALLS)}" + ) +except ValueError as exc: + print(f"function node over a pod stream, run(load_only=True) refused: {exc}") +print(f" upstream function calls: {len(CALLS)}") diff --git a/superpowers/scripts/itl_1121/c8_cr388_load_only_delivery_paths.py b/superpowers/scripts/itl_1121/c8_cr388_load_only_delivery_paths.py new file mode 100644 index 000000000..47970af74 --- /dev/null +++ b/superpowers/scripts/itl_1121/c8_cr388_load_only_delivery_paths.py @@ -0,0 +1,96 @@ +"""PR #388 CodeRabbit round 1: more ways a load-only run could read a delivering stream. + +D: a job's source slot bound to a SideEffectPodStream; job.run(load_only=True). +H: a stream outside a job that wraps a SideEffectPodStream (an operator over it). +C: a standalone OperatorJobNode without a database, run(load_only=True): its status. +""" + +from typing import Any + +import pyarrow as pa + +from orcapod import PipelineJob, function_pod +from orcapod.core.nodes.operator_node import OperatorJobNode +from orcapod.core.operators import MapData +from orcapod.core.streams import ArrowTableStream +from orcapod.databases import InMemoryArrowDatabase +from orcapod.side_effects import tap_pod +from orcapod.sources import DictSource +from orcapod.types import CacheMode + +UP: list[int] = [] + + +def deliver_up(x: int, ctx: Any) -> None: + UP.append(x) + + +@function_pod(output_keys="w") +def twice(x: int) -> int: + return 2 * x + + +schema = pa.schema( + [ + pa.field("k", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] +) + + +def base() -> Any: + return ArrowTableStream( + pa.table({"k": [1, 2], "x": [1, 2]}, schema=schema), tag_columns=["k"] + ) + + +def delivering() -> Any: + tap: Any = tap_pod(deliver_up) + return tap.process(base()) + + +# D +db = InMemoryArrowDatabase() +job = PipelineJob(name="j", database=db) +with job: + src = DictSource( + [{"k": 1, "x": 1}, {"k": 2, "x": 2}], tag_columns=["k"], source_id="s" + ) + twice.pod(src, label="tw") +job.run() +UP.clear() +try: + job.bind(sources={"s": delivering()}) + job.run(load_only=True) + print( + f"D: bound source slot, job.run(load_only=True): upstream delivered {len(UP)}" + ) +except Exception as exc: + print(f"D: {type(exc).__name__}: {exc}") + +# H +UP.clear() +wrapped: Any = MapData({"x": "x2"})(delivering()) # an operator stream outside a job +print("H: wrapped input type:", type(wrapped).__name__) +node = OperatorJobNode( + operator=MapData({"x2": "x3"}), + input_streams=(wrapped,), + pipeline_database=InMemoryArrowDatabase(), + cache_mode=CacheMode.OFF, +) +try: + node.run(load_only=True) + print( + f"H: operator node over the wrapped stream, run(load_only=True): upstream delivered {len(UP)}" + ) +except Exception as exc: + print(f"H: {type(exc).__name__}: {exc}") + +# C +op = OperatorJobNode( + operator=MapData({"x": "y"}), input_streams=(base(),), pipeline_database=None +) +view = op.run(load_only=True) +print( + f"C: no-database operator, run(load_only=True): status {view.status}; full run holds it: {op.holds_current_run(load_only=False)}" +) diff --git a/superpowers/scripts/itl_1121/c9_interrupted_log_commit.py b/superpowers/scripts/itl_1121/c9_interrupted_log_commit.py new file mode 100644 index 000000000..66f879930 --- /dev/null +++ b/superpowers/scripts/itl_1121/c9_interrupted_log_commit.py @@ -0,0 +1,57 @@ +"""PR #381 summary finding: a standalone side-effect job node (input outside a job, store +attached) holds its rows before _run_alone commits the invocation log. A KeyboardInterrupt +during that commit skips the Exception-only cleanup, so the node keeps a current run: the +next run() is held and never retries the commit.""" + +from typing import Any +from unittest import mock + +import pyarrow as pa + +from orcapod.core.streams import ArrowTableStream +from orcapod.databases import InMemoryArrowDatabase +from orcapod.side_effects import SideEffectJobNode, tap_pod + +DELIVERED: list[int] = [] + + +def deliver(x: int, ctx: Any) -> None: + DELIVERED.append(x) + + +schema = pa.schema( + [ + pa.field("k", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] +) +stream = ArrowTableStream( + pa.table({"k": [1, 2], "x": [1, 2]}, schema=schema), tag_columns=["k"] +) +db = InMemoryArrowDatabase() +tap: Any = tap_pod(deliver) +node = SideEffectJobNode(tap, stream) +node.attach_databases(db) + +with mock.patch.object(InMemoryArrowDatabase, "flush", side_effect=KeyboardInterrupt): + try: + node.run() + except KeyboardInterrupt: + print("run(): KeyboardInterrupt during the log commit") +print( + f"after the interrupted commit: holds_current_run={node.holds_current_run(load_only=False)} is_stale={node.is_stale} delivered={len(DELIVERED)}" +) +flushes: list[int] = [] +real_flush = InMemoryArrowDatabase.flush + + +def counting_flush(self: Any) -> None: + flushes.append(1) + real_flush(self) + + +with mock.patch.object(InMemoryArrowDatabase, "flush", counting_flush): + node.run() +print( + f"next run(): commits attempted={len(flushes)} delivered in total={len(DELIVERED)}" +) diff --git a/test-objective/integration/test_caching_flows.py b/test-objective/integration/test_caching_flows.py index ed91d25e4..34d6a051b 100644 --- a/test-objective/integration/test_caching_flows.py +++ b/test-objective/integration/test_caching_flows.py @@ -23,7 +23,11 @@ # --------------------------------------------------------------------------- +_DOUBLE_CALLS: list[int] = [] + + def _double(x: int) -> int: + _DOUBLE_CALLS.append(x) return x * 2 @@ -86,9 +90,11 @@ def test_second_run_uses_cache(self): result_database=result_db, ) # A node shows rows only after a run (ITL-1095); this run reads them all. + _DOUBLE_CALLS.clear() node2.run() data = list(node2.iter_data()) assert len(data) == 3 + assert _DOUBLE_CALLS == [] # every result came from the store class TestDerivedSourceReingestion: diff --git a/tests/test_pipeline/test_node_view_review_fixes.py b/tests/test_pipeline/test_node_view_review_fixes.py new file mode 100644 index 000000000..b3d90e9f9 --- /dev/null +++ b/tests/test_pipeline/test_node_view_review_fixes.py @@ -0,0 +1,683 @@ +"""Defects in the job node current-run view found by a full review (ITL-1121). + +The job node view (ITL-1095): a job node shows its current run; a run that raises leaves the +nodes it ran empty and stale; a node that holds a current run is not run again; a load-only +run computes, delivers and writes nothing. Each test reproduces one review comment through the +public interface (``superpowers/scripts/itl_1121`` holds the scripts). +""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.pipeline.observer import NoOpObserver +from orcapod.side_effects import SideEffectPod, tap_pod +from orcapod.sources import DictSource + +ORCHESTRATORS = {"sync": SyncPipelineOrchestrator, "async": AsyncPipelineOrchestrator} + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + """Return ``x + 1``.""" + return x + 1 + + +def _tap(fn: Any) -> SideEffectPod: + """A tap side-effect pod over ``fn``.""" + pod = tap_pod(fn) + assert isinstance(pod, SideEffectPod) + return pod + + +def _source() -> DictSource: + """Three rows, tag ``k``, data ``x``.""" + return DictSource( + [{"k": i, "x": i} for i in range(3)], tag_columns=["k"], source_id="s" + ) + + +# --------------------------------------------------------------------------- +# Comment 1: a function node is stamped before on_node_end +# --------------------------------------------------------------------------- + + +class _ReadingObserver(NoOpObserver): + """Reads each job node in ``on_node_end``: the row count it sees, by label.""" + + def __init__(self, job: PipelineJob) -> None: + self.job = job + self.seen: dict[str, int] = {} + + def contextualize(self, *identity_path: str) -> _ReadingObserver: + """The same observer for every node.""" + return self + + def on_node_end(self, node_label: str, node_hash: str) -> None: + """Read the node that just ended.""" + node = self.job.nodes.get(node_label) + if node is not None: + self.seen[node_label] = node.as_table().num_rows + + +def _rowwise_job(db: Any) -> PipelineJob: + """``rowwise`` = ``inc`` over three rows.""" + job = PipelineJob(name="j", database=db) + with job: + inc.pod(_source(), label="rowwise") + return job + + +@pytest.mark.parametrize("load_only", [False, True]) +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) +def test_an_observer_reading_a_node_in_on_node_end_sees_its_rows( + orchestrator: str, load_only: bool +) -> None: + """The node is current when ``on_node_end`` fires, and keeps its rows after the run.""" + db = InMemoryArrowDatabase() + if load_only: + _rowwise_job(db).run() # store the results a load-only run loads + job = _rowwise_job(db) + observer = _ReadingObserver(job) + job.run( + orchestrator=ORCHESTRATORS[orchestrator](), + observer=observer, + load_only=load_only, + ) + node = job.nodes["rowwise"] + assert observer.seen["rowwise"] == 3 + assert node.as_table().num_rows == 3 + assert node.holds_current_run(load_only=load_only) + + +# --------------------------------------------------------------------------- +# Comment 2: an operator node's lazy output that fails to materialize clears the node +# --------------------------------------------------------------------------- + +_FAIL = {"on": True} + + +@function_pod(output_keys="y") +def flaky(x: int) -> int: + """Raise while ``_FAIL["on"]``; else return ``x + 1``.""" + if _FAIL["on"]: + raise RuntimeError("input failed") + return x + 1 + + +def _base_stream() -> Any: + """Two rows, tag ``id`` and data ``x``, both non-nullable ``int64``.""" + import pyarrow as pa + + from orcapod.core.streams import ArrowTableStream + + schema = pa.schema( + [ + pa.field("id", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] + ) + return ArrowTableStream( + pa.table({"id": [1, 2], "x": [1, 2]}, schema=schema), tag_columns=["id"] + ) + + +@pytest.mark.parametrize("with_database", [False, True]) +def test_an_operator_run_whose_lazy_output_raises_holds_no_run( + with_database: bool, +) -> None: + """No database, or ``CacheMode.OFF``: the output is computed when the view is built; + if that raises, the node reads as one with no run, and the next run computes.""" + from orcapod.core.nodes.operator_node import OperatorJobNode + from orcapod.core.operators import MapData + from orcapod.types import CacheMode + + _FAIL["on"] = True + node = OperatorJobNode( + operator=MapData({"y": "z"}), + input_streams=(flaky.pod(_base_stream()),), + pipeline_database=InMemoryArrowDatabase() if with_database else None, + cache_mode=CacheMode.OFF, + ) + with pytest.raises(RuntimeError, match="input failed"): + node.run() + assert not node.holds_current_run(load_only=False) + assert node.is_stale + assert node.as_table().num_rows == 0 + _FAIL["on"] = False + assert node.run().as_table().num_rows == 2 + + +# --------------------------------------------------------------------------- +# Comment 4: the summary says "nothing to do" only when the run ran nothing +# --------------------------------------------------------------------------- + +_DELIVERED: list[int] = [] +_NOTHING_TO_DO = "nothing to do; every node holds a current run" + + +def _deliver(y: int, ctx: Any) -> None: + """A side effect: record the delivered value.""" + _DELIVERED.append(y) + + +def _job_below_rowwise(below: str) -> PipelineJob: + """``rowwise`` (``inc``) with a side-effect node or an operator node ``below`` it.""" + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + rows: Any = inc.pod(_source(), label="rowwise") + if below == "side_effect": + _tap(_deliver).process(rows, label="below") + else: + rows.select_data_columns(["y"], label="below") + return job + + +def _run_job_or_node(job: PipelineJob, how: str) -> Any: + """Run ``job`` on an orchestrator, or run its node ``below``; return the summary.""" + if how == "node.run": + job.nodes["below"].run() + return job.nodes["below"].last_run + return job.run(orchestrator=ORCHESTRATORS[how]()) + + +@pytest.mark.parametrize("how", ["sync", "async", "node.run"]) +@pytest.mark.parametrize("below", ["side_effect", "operator"]) +def test_a_run_that_ran_a_cleared_node_is_not_nothing_to_do( + below: str, how: str +) -> None: + """A node cleared below a held function node runs again (a side effect delivers); + the summary says what ran, not that there was nothing to do.""" + _DELIVERED.clear() + job = _job_below_rowwise(below) + _run_job_or_node(job, how) + job.nodes["below"].clear_cache() + _DELIVERED.clear() + summary = _run_job_or_node(job, how) + if below == "side_effect": + assert len(_DELIVERED) == 3 + assert job.nodes["below"].holds_current_run(load_only=False) + assert summary.nodes["rowwise"].held == 3 + assert _NOTHING_TO_DO not in repr(summary) + assert repr(summary).splitlines()[0].endswith("all rows succeeded.") + + +# A node.run() of a node that holds a current run returns at once and makes no summary +# (node.last_run stays the previous one), so "nothing ran" is a job run's case. +@pytest.mark.parametrize("how", ["sync", "async"]) +@pytest.mark.parametrize("below", ["side_effect", "operator"]) +def test_a_run_that_ran_nothing_is_nothing_to_do(below: str, how: str) -> None: + """A second run of an unchanged job runs nothing and says so.""" + job = _job_below_rowwise(below) + _run_job_or_node(job, how) + summary = _run_job_or_node(job, how) + assert _NOTHING_TO_DO in repr(summary).splitlines()[0] + + +@pytest.mark.parametrize("how", ["sync", "async", "node.run"]) +def test_a_job_with_no_function_node_says_what_ran(how: str) -> None: + """Only an operator: the first run succeeds; a second job run has nothing to do.""" + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + _source().select_data_columns(["x"], label="below") + first = _run_job_or_node(job, how) + assert repr(first).splitlines()[0].endswith("all rows succeeded.") + if how != "node.run": # a held node.run() makes no summary (see above) + second = _run_job_or_node(job, how) + assert _NOTHING_TO_DO in repr(second).splitlines()[0] + + +# --------------------------------------------------------------------------- +# Comment 5: a load-only execute() whose input raises clears the side-effect node +# --------------------------------------------------------------------------- + + +@function_pod(output_keys="z") +def _boom(y: int) -> int: + """Always raise.""" + raise RuntimeError("input failed") + + +def _failing_y_stream() -> Any: + """A pod stream outside a job (tag ``k``, data ``z``) that raises when read.""" + import pyarrow as pa + + from orcapod.core.streams import ArrowTableStream + + schema = pa.schema( + [ + pa.field("k", pa.int64(), nullable=False), + pa.field("y", pa.int64(), nullable=False), + ] + ) + return _boom.pod( + ArrowTableStream( + pa.table({"k": [1], "y": [1]}, schema=schema), tag_columns=["k"] + ) + ) + + +class _FailingChannel: + """An input channel whose iteration raises.""" + + def __aiter__(self) -> _FailingChannel: + return self + + async def __anext__(self) -> Any: + raise RuntimeError("input failed") + + +@pytest.mark.parametrize("path", ["sync", "async"]) +@pytest.mark.parametrize("load_only", [False, True]) +def test_an_execute_whose_input_raises_leaves_the_side_effect_node_stale( + path: str, load_only: bool +) -> None: + """The node and the node below it read as not run, whatever the run kind. + + ``execute``/``async_execute`` are the node's run entry points (orchestrators call + them); this test is about them, so it calls them directly. + """ + import asyncio + + from orcapod.channels import Channel + + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + rows = inc.pod(_source(), label="rowwise") + tapped: Any = _tap(_deliver).process(rows, label="tap") + tapped.select_data_columns(["y"], label="down") + job.run() + tap, down = job.nodes["tap"], job.nodes["down"] + # The async delivery path reads its input inside a task group, which raises the + # input's error in an ExceptionGroup. + with pytest.raises((RuntimeError, ExceptionGroup)): + if path == "sync": + tap.execute(_failing_y_stream(), load_only=load_only) + else: + out: Channel[Any] = Channel() + asyncio.run( + tap.async_execute([_FailingChannel()], out.writer, load_only=load_only) + ) + assert tap.is_stale and down.is_stale + assert tap.as_table().num_rows == 0 + assert down.as_table().num_rows == 0 + + +# --------------------------------------------------------------------------- +# Comment 6: a load-only run refuses an input that delivers when it is read +# --------------------------------------------------------------------------- + +_UPSTREAM_DELIVERED: list[int] = [] + + +def _deliver_upstream(x: int, ctx: Any) -> None: + """The side effect of the stream outside a job: record the delivered value.""" + _UPSTREAM_DELIVERED.append(x) + + +def _deliver_own(x: int, ctx: Any) -> None: + """The side effect of the node under test: record the delivered value.""" + _DELIVERED.append(x) + + +@function_pod(output_keys="w") +def _twice(x: int) -> int: + """Return ``2 * x``.""" + return 2 * x + + +# The decorated function's pod, typed for a node's constructor. +_twice_pod: Any = _twice.pod + + +def _delivering_stream() -> Any: + """A side-effect pod's stream outside a job: reading it delivers each row.""" + return _tap(_deliver_upstream).process(_base_stream()) + + +def _node_over(kind: str, stream: Any) -> Any: + """A job node of ``kind`` built directly over ``stream`` (no job).""" + from orcapod.core.nodes.function_node import FunctionJobNode + from orcapod.core.nodes.operator_node import OperatorJobNode + from orcapod.core.operators import MapData + from orcapod.side_effects import SideEffectJobNode + from orcapod.types import CacheMode + + if kind == "side_effect": + return SideEffectJobNode(_tap(_deliver_own), stream) + if kind == "function": + return FunctionJobNode( + _twice_pod, stream, pipeline_database=InMemoryArrowDatabase() + ) + return OperatorJobNode( + operator=MapData({"x": "renamed"}), + input_streams=(stream,), + pipeline_database=InMemoryArrowDatabase(), + cache_mode=CacheMode.OFF, + ) + + +@pytest.mark.parametrize("kind", ["side_effect", "function", "operator"]) +def test_a_load_only_run_refuses_an_input_that_delivers(kind: str) -> None: + """Refused before the input is read: nothing is delivered and the node holds no run.""" + _UPSTREAM_DELIVERED.clear() + _DELIVERED.clear() + node = _node_over(kind, _delivering_stream()) + with pytest.raises(ValueError, match="delivers its side effects when it is read"): + node.run(load_only=True) + assert _UPSTREAM_DELIVERED == [] + assert _DELIVERED == [] + assert not node.holds_current_run(load_only=True) + + +@pytest.mark.parametrize("kind", ["side_effect", "function", "operator"]) +def test_a_load_only_cascade_refuses_an_input_that_delivers(kind: str) -> None: + """The input that delivers feeds a node the cascade would run first.""" + from orcapod.core.nodes.operator_node import OperatorJobNode + from orcapod.core.operators import DropNulls + from orcapod.types import CacheMode + + _UPSTREAM_DELIVERED.clear() + first = _node_over(kind, _delivering_stream()) + target = OperatorJobNode( + operator=DropNulls(), + input_streams=(first,), + pipeline_database=InMemoryArrowDatabase(), + cache_mode=CacheMode.OFF, + ) + with pytest.raises(ValueError, match="delivers its side effects when it is read"): + target.run(load_only=True) + assert _UPSTREAM_DELIVERED == [] + assert not first.holds_current_run(load_only=True) + assert not target.holds_current_run(load_only=True) + + +@pytest.mark.parametrize("kind", ["side_effect", "function", "operator"]) +def test_a_full_run_over_an_input_that_delivers_reads_it(kind: str) -> None: + """Unchanged: a full run reads the stream outside a job, which delivers.""" + _UPSTREAM_DELIVERED.clear() + node = _node_over(kind, _delivering_stream()) + assert node.run().as_table().num_rows == 2 + assert len(_UPSTREAM_DELIVERED) == 2 + + +def _job_over_source_bound_to_a_delivering_stream() -> PipelineJob: + """``tw`` (``_twice``) over source slot ``s``, run once, then bound to a stream that + delivers when read.""" + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + source = DictSource( + [{"id": 1, "x": 1}, {"id": 2, "x": 2}], tag_columns=["id"], source_id="s" + ) + _twice_pod(source, label="tw") + job.run() + job.bind(sources={"s": _delivering_stream()}) + return job + + +@pytest.mark.parametrize("how", ["sync", "async", "node.run", "source.run"]) +def test_a_load_only_run_refuses_a_source_slot_bound_to_a_delivering_stream( + how: str, +) -> None: + """A source slot reads its bound stream: refused before it is read.""" + _UPSTREAM_DELIVERED.clear() + job = _job_over_source_bound_to_a_delivering_stream() + with pytest.raises(ValueError, match="delivers its side effects when it is read"): + if how == "node.run": + job.nodes["tw"].run(load_only=True) + elif how == "source.run": + job.nodes["tw"].upstreams[0].run(load_only=True) + else: + job.run(orchestrator=ORCHESTRATORS[how](), load_only=True) + assert _UPSTREAM_DELIVERED == [] + + +@pytest.mark.parametrize("kind", ["side_effect", "function", "operator"]) +def test_a_load_only_run_refuses_a_stream_that_reads_a_delivering_stream( + kind: str, +) -> None: + """An operator's stream outside a job over a side-effect stream delivers when read.""" + _UPSTREAM_DELIVERED.clear() + stream: Any = _delivering_stream().select_data_columns(["x"]) + node = _node_over(kind, stream) + with pytest.raises(ValueError, match="delivers its side effects when it is read"): + node.run(load_only=True) + assert _UPSTREAM_DELIVERED == [] + + +def test_a_load_only_operator_run_without_a_database_is_loaded_only() -> None: + """Its rows come from a load-only run: shown as such, and not a full run's result.""" + from orcapod.core.nodes.operator_node import OperatorJobNode + from orcapod.core.operators import MapData + from orcapod.core.streams import ViewStatus + + node = OperatorJobNode( + operator=MapData({"x": "y"}), + input_streams=(_base_stream(),), + pipeline_database=None, + ) + assert node.run(load_only=True).status == ViewStatus.LOADED_ONLY + assert node.holds_current_run(load_only=True) + assert not node.holds_current_run(load_only=False) + assert node.run().status == ViewStatus.ROWS + + +def test_an_interrupted_log_commit_leaves_the_side_effect_node_without_a_run() -> None: + """A standalone side-effect node with a store: a KeyboardInterrupt while its run commits + the invocation log clears the node, so the next run runs (and commits) again.""" + from unittest import mock + + from orcapod.side_effects import SideEffectJobNode + + _DELIVERED.clear() + node = SideEffectJobNode(_tap(_deliver_own), _base_stream()) + node.attach_databases(InMemoryArrowDatabase()) + with ( + mock.patch.object( + InMemoryArrowDatabase, "flush", side_effect=KeyboardInterrupt + ), + pytest.raises(KeyboardInterrupt), + ): + node.run() + assert not node.holds_current_run(load_only=False) + assert node.is_stale + with mock.patch.object( + InMemoryArrowDatabase, "flush", autospec=True, side_effect=lambda self: None + ) as flush: + node.run() + assert flush.call_count >= 1 + + +# --------------------------------------------------------------------------- +# Developer decision (November 14, 2026): a load-only run also refuses an input that +# computes when read ("Refuse is consistent") +# --------------------------------------------------------------------------- + +_UPSTREAM_CALLS: list[int] = [] +_COMPUTES = "computes when it is read" + + +@function_pod(output_keys="x") +def _recorded_copy(x: int) -> int: + """Return ``x``, recording the call.""" + _UPSTREAM_CALLS.append(x) + return x + + +def _computing_stream(kind: str) -> Any: + """A stream outside a job that computes ``_recorded_copy`` when read. + + ``pod``: the function pod's stream; ``cached``: a cached function pod's stream over + an empty store; ``wrapped``: an operator's stream outside a job over the pod's + stream. (A ``FunctionNode`` outside a job is a blueprint: reading it raises.) + """ + from orcapod.core.cached_function_pod import CachedFunctionPod + + base = _base_stream() + pod: Any = _recorded_copy.pod + if kind == "pod": + return pod(base) + if kind == "cached": + return CachedFunctionPod(pod, result_database=InMemoryArrowDatabase())(base) + stream: Any = pod(base) + return stream.select_data_columns(["x"]) + + +@pytest.mark.parametrize("stream_kind", ["pod", "cached", "wrapped"]) +@pytest.mark.parametrize("kind", ["side_effect", "function", "operator"]) +def test_a_load_only_run_refuses_an_input_that_computes( + kind: str, stream_kind: str +) -> None: + """Refused before the input is read: its function is never called.""" + _UPSTREAM_CALLS.clear() + node = _node_over(kind, _computing_stream(stream_kind)) + with pytest.raises(ValueError, match=_COMPUTES): + node.run(load_only=True) + assert _UPSTREAM_CALLS == [] + assert not node.holds_current_run(load_only=True) + + +@pytest.mark.parametrize("kind", ["side_effect", "function", "operator"]) +def test_a_load_only_cascade_refuses_an_input_that_computes(kind: str) -> None: + """The computing input feeds a node the cascade would run first.""" + from orcapod.core.nodes.operator_node import OperatorJobNode + from orcapod.core.operators import DropNulls + from orcapod.types import CacheMode + + _UPSTREAM_CALLS.clear() + first = _node_over(kind, _computing_stream("pod")) + target = OperatorJobNode( + operator=DropNulls(), + input_streams=(first,), + pipeline_database=InMemoryArrowDatabase(), + cache_mode=CacheMode.OFF, + ) + with pytest.raises(ValueError, match=_COMPUTES): + target.run(load_only=True) + assert _UPSTREAM_CALLS == [] + + +@pytest.mark.parametrize("how", ["sync", "async", "node.run", "source.run"]) +def test_a_load_only_run_refuses_a_source_slot_bound_to_a_computing_stream( + how: str, +) -> None: + """A source slot bound to a function pod's stream outside a job: refused.""" + _UPSTREAM_CALLS.clear() + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + source = DictSource( + [{"id": 1, "x": 1}, {"id": 2, "x": 2}], tag_columns=["id"], source_id="s" + ) + _twice_pod(source, label="tw") + job.run() + job.bind(sources={"s": _computing_stream("pod")}) + with pytest.raises(ValueError, match=_COMPUTES): + if how == "node.run": + job.nodes["tw"].run(load_only=True) + elif how == "source.run": + job.nodes["tw"].upstreams[0].run(load_only=True) + else: + job.run(orchestrator=ORCHESTRATORS[how](), load_only=True) + assert _UPSTREAM_CALLS == [] + + +@pytest.mark.parametrize("kind", ["side_effect", "function", "operator"]) +def test_a_full_run_over_an_input_that_computes_reads_it(kind: str) -> None: + """Unchanged: a run without ``load_only`` reads (computes) the outside stream.""" + _UPSTREAM_CALLS.clear() + node = _node_over(kind, _computing_stream("pod")) + assert node.run().as_table().num_rows == 2 + assert len(_UPSTREAM_CALLS) == 2 + + +# --------------------------------------------------------------------------- +# PR #388 review round 2: what the load-only refusal walks through +# --------------------------------------------------------------------------- + + +def _held_table_over_a_computing_stream() -> Any: + """An ``ArrowTableStream`` holding its rows, whose ``upstreams`` name a function pod's + stream outside a job; reading it reads only the held table.""" + import pyarrow as pa + + from orcapod.core.streams import ArrowTableStream + + schema = pa.schema( + [ + pa.field("id", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] + ) + return ArrowTableStream( + pa.table({"id": [1, 2], "x": [1, 2]}, schema=schema), + tag_columns=["id"], + upstreams=(_computing_stream("pod"),), + ) + + +@pytest.mark.parametrize("kind", ["side_effect", "function", "operator"]) +def test_a_load_only_run_reads_a_table_stream_whatever_its_upstreams( + kind: str, +) -> None: + """An ``ArrowTableStream`` neither computes nor delivers: not refused.""" + _UPSTREAM_CALLS.clear() + node = _node_over(kind, _held_table_over_a_computing_stream()) + node.run(load_only=True) + assert node.holds_current_run(load_only=True) + assert _UPSTREAM_CALLS == [] + + +def _blueprint_delivering_node() -> Any: + """A ``SideEffectNode`` outside a job: reading it delivers each row.""" + from orcapod.side_effects import SideEffectNode + + return SideEffectNode(_tap(_deliver_upstream), _base_stream()) + + +@pytest.mark.parametrize("kind", ["side_effect", "function", "operator"]) +def test_a_load_only_run_refuses_a_side_effect_node_outside_a_job(kind: str) -> None: + """Refused before the input is read: nothing is delivered.""" + _UPSTREAM_DELIVERED.clear() + node = _node_over(kind, _blueprint_delivering_node()) + with pytest.raises(ValueError, match="delivers its side effects when it is read"): + node.run(load_only=True) + assert _UPSTREAM_DELIVERED == [] + assert not node.holds_current_run(load_only=True) + + +@pytest.mark.parametrize("how", ["sync", "async", "node.run", "source.run"]) +def test_a_load_only_run_refuses_a_source_slot_bound_to_a_side_effect_node( + how: str, +) -> None: + """A source slot bound to a ``SideEffectNode`` outside a job: refused.""" + _UPSTREAM_DELIVERED.clear() + job = PipelineJob(name="j", database=InMemoryArrowDatabase()) + with job: + source = DictSource( + [{"id": 1, "x": 1}, {"id": 2, "x": 2}], tag_columns=["id"], source_id="s" + ) + _twice_pod(source, label="tw") + job.run() + job.bind(sources={"s": _blueprint_delivering_node()}) + with pytest.raises(ValueError, match="delivers its side effects when it is read"): + if how == "node.run": + job.nodes["tw"].run(load_only=True) + elif how == "source.run": + job.nodes["tw"].upstreams[0].run(load_only=True) + else: + job.run(orchestrator=ORCHESTRATORS[how](), load_only=True) + assert _UPSTREAM_DELIVERED == [] + + +@pytest.mark.parametrize("kind", ["side_effect", "function", "operator"]) +def test_a_full_run_over_a_side_effect_node_outside_a_job_reads_it(kind: str) -> None: + """Unchanged: a run without ``load_only`` reads it, which delivers.""" + _UPSTREAM_DELIVERED.clear() + node = _node_over(kind, _blueprint_delivering_node()) + assert node.run().as_table().num_rows == 2 + assert len(_UPSTREAM_DELIVERED) == 2