diff --git a/.zed/rules b/.zed/rules index 0342f7342..2d923a544 100644 --- a/.zed/rules +++ b/.zed/rules @@ -773,8 +773,11 @@ data. The rules: by the user is never changed. SourceStreamBuilder.build has no bare-hash default: it needs a source_id or a source_kind. - ArrowTableStream requires at least one data column; raises ValueError otherwise. -- FunctionNode Phase 1 returns ALL records in the shared pipeline_path DB table. - Phase 2 skips inputs whose hash is already in the DB. +- After a run, a FunctionJobNode returns exactly the rows of that run's inputs, sync or + async (ITL-1093; a run that raised defines no rows). Read without a completed run in this + process (a fresh or loaded job), it returns every record stored under its content hash for + the current data context, not only its current inputs. A run skips inputs whose entry is + already stored. - Empty data → ArrowTableSource raises ValueError("Table is empty"). - DerivedSource over a job node before run() → presents zero rows with the origin's columns (no computed records; no error), in the plain form and with tags_as_data. diff --git a/CLAUDE.md b/CLAUDE.md index 8164ba5f9..d075910b4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -814,9 +814,11 @@ data. The rules: by the user is never changed. `SourceStreamBuilder.build` has no bare-hash default: it needs a `source_id` or a `source_kind`. - `ArrowTableStream` requires at least one data column; raises `ValueError` otherwise. -- `FunctionNode.iter_data()` Phase 1 returns ALL records in the shared `pipeline_path` - DB table (not filtered to current inputs). Phase 2 skips inputs whose hash is already - in the DB. +- After a run, a `FunctionJobNode` returns exactly the rows of that run's inputs, sync or + async (ITL-1093; a run that raised defines no rows). Read without a completed run in this + process (a fresh or loaded job), it returns every record stored under its content hash for + the current data context, not only its current inputs. A run skips inputs whose entry is + already stored. - Empty data → `ArrowTableSource` raises `ValueError("Table is empty")`. - `DerivedSource` over a job node before `run()` → presents zero rows with the origin's columns (no computed records; no error), in the plain form and with `tags_as_data`. diff --git a/changelog.d/ITL-1093.md b/changelog.d/ITL-1093.md new file mode 100644 index 000000000..2e4421eb8 --- /dev/null +++ b/changelog.d/ITL-1093.md @@ -0,0 +1,3 @@ +### Fixed + +- After `job.run()` on either orchestrator, a function job node's `as_table()`, `iter_data()`, `head()` and `run()`, and a stream built over the node, return exactly the rows of that run's inputs (ITL-1093). Before, after an async run a node returned every row stored under its key: a pipeline whose source rows changed between runs under one `source_id` showed the results of earlier runs too (for a function over `group_by`, the partial groups of an earlier run next to the full ones; rows of source rows no longer present). The sync orchestrator did the same when one job object ran again with its source rebound. Rows sent downstream during the run and the `RunSummary` were already right. Results of other inputs stay stored and are reused when those inputs come back; nothing stored, no hash and no table layout changes. A run that raised defines no rows: the node then reads as a node with no run in this process. A node read with no finished run in this process (a job built but not run, or a loaded job) still returns every row stored under its key. diff --git a/design_issues/ITL-1093-JR1.md b/design_issues/ITL-1093-JR1.md new file mode 100644 index 000000000..e270f4ffc --- /dev/null +++ b/design_issues/ITL-1093-JR1.md @@ -0,0 +1,26 @@ +## `src/orcapod/core/nodes/function_node.py` + +### JR1 — After a run, a function job node returned rows of other runs' inputs +**Status:** resolved +**Severity:** high +**Issue:** ITL-1093 + +A `FunctionJobNode` kept one in-memory cache of results keyed by base entry id, and every read +(`iter_data`, and through it `as_table`, `run()` and any consumer; `head`) returned all of it. +Nothing recorded which inputs the run processed. `async_execute` loads every pipeline record +stored under the node's content hash into that cache before routing its inputs, so after an +async run the node returned every row ever stored for it: partial groups of an earlier run next +to full ones, rows of source rows no longer present. The sync `execute` loaded only its inputs' +entries, so it looked right with a new job per run, but a cache is add-only: running the same +node objects again (one job, its source rebound) accumulated rows on the sync path too. Rows sent +downstream during the run and the `RunSummary` were right on both paths. + +**Fix:** The node records the base entry ids of its last completed run's inputs +(`_run_entry_counts`, set by `execute` and `async_execute`), and every read of its rows goes +through one helper, `_held_pairs`, which keeps only those inputs' rows (one per input, so two +equal inputs give two rows, as downstream received them). The cache stays a superset: stored +results of other inputs stay stored and reusable. A run that raised defines no rows: the node +clears its in-memory rows and reads as a node with no run in this process (every row stored +under its content hash), on both orchestrators. Nothing stored, no hash and no layout changed. +A read with no completed run in this process (a fresh or loaded job) still returns every row +stored under the node's content hash. diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 108453b24..5f414b039 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -333,6 +333,13 @@ database. Each recorded step becomes a node, available afterwards as `job.nodes[ listed under its `source_id`). A node is a stream, so `as_table()`, `iter_data()` and `output_schema()` all work on it. +After `job.run()`, a function node returns exactly the rows of that run's inputs, on either +orchestrator. The database may hold more under the same node: results of an earlier run +over other rows of the source stay stored and are reused when those rows come back, but +they are not rows of this run. A run that raised defines no rows. A node read with no +finished run in this process (a job built but not run, or a loaded job) returns every row +stored under its key. + `InMemoryArrowDatabase` lasts only as long as the Python process. Use `DeltaTableDatabase("some/directory")` from `orcapod.databases` to keep results on disk between sessions. @@ -696,8 +703,9 @@ file names each run's rows by the content hash the run stored them under, so a l reads exactly what the same node built from your code would read from that store, and no rows under that key is an empty result, not an error. -- A function node's `as_table()`, `iter_data()` and `get_all_records()` return the rows the - saved run stored, with the types they were stored with. Rows come in the order the store +- A function node's `as_table()`, `iter_data()` and `get_all_records()` return the rows + stored under the saved run's key, with the types they were stored with: those of every + run of that node over the same sources, not only the saved run's inputs. Rows come in the order the store returns them; sort them if order matters. - Operators (a join, a filter) store outputs only when caching is enabled, under `odb_v1`. The saved file does not record an operator's cache mode (follow-up ITL-977), so attach a diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 186a8e384..54f83f8b4 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -21,7 +21,8 @@ import logging import uuid from abc import abstractmethod -from collections.abc import Callable, Iterator, Mapping, Sequence +from collections import Counter +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Literal, NamedTuple, cast from orcapod import contexts @@ -1054,6 +1055,10 @@ class FunctionJobNode(FunctionNodeBase): _current_tables_checked: bool = False # Set once a store that cannot list its tables was reported. _unlisted_store_reported: bool = False + # Base entry ids of the inputs of the last completed run in this process, each with + # its number of inputs; None when no run completed (ITL-1093). A class default so + # nodes rebuilt from a saved descriptor start without a run. + _run_entry_counts: Counter[bytes] | None = None def __init__( self, @@ -1083,6 +1088,7 @@ def __init__( ] = {} self._cached_output_table: "pa.Table | None" = None self._cached_content_hash_column: "pa.Array | None" = None + self._run_entry_counts = None # DB persistence state (initially None; set via __init__ params or attach_databases) self._pipeline_database: ArrowDatabaseProtocol | None = None @@ -1505,11 +1511,44 @@ def from_descriptor( # ------------------------------------------------------------------ def clear_cache(self) -> None: - """Clear in-memory output caches, content hash cache, and node identity path cache.""" + """Clear in-memory output caches, content hash cache, and node identity path cache. + + The node then holds no run: it reads as a node with no completed run in this + process (every row stored under its content hash). + """ super().clear_cache() self._cached_output_datas.clear() self._cached_output_table = None self._cached_content_hash_column = None + self._run_entry_counts = None + + def _held_pairs(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: + """Yield the (tag, data) rows this node holds; every read of them goes through here. + + After a completed run, the rows are those of that run's inputs: one per input + (two equal inputs give two rows, as downstream received them). The in-memory + cache may hold more (results of other inputs, read from the store), which stay + cached for reuse but are not rows of this run (ITL-1093). Without a completed + run in this process, every held row. Rows come in the order the cache holds + them. An input without output (an inactive function) is not a row. + """ + held = self._cached_output_datas + counts = self._run_entry_counts + if counts is None: + pairs: Iterable[tuple[TagProtocol, DataProtocol | None]] = held.values() + else: + pairs = ( + pair for eid, pair in held.items() for _ in range(counts.get(eid, 0)) + ) + for tag, data in pairs: + if data is not None: + yield tag, data + + def _set_run_rows(self, entry_ids: list[bytes]) -> None: + """Make the inputs of a completed run, by base entry id, the rows this node holds.""" + self._run_entry_counts = Counter(entry_ids) + self._cached_output_table = None + self._cached_content_hash_column = None def set_ephemeral_store(self, store: "ArrowDatabaseProtocol | None") -> None: """Assign or remove the ephemeral result store. @@ -1872,6 +1911,11 @@ def execute( (populated from DB if needed), compute if missing, fire ``on_data_end`` or ``on_data_crash``. + Afterwards the node's rows (``iter_data``, ``as_table``, ``head``) are those + of this run's inputs. If the run raises, it defines no rows: the node drops + its in-memory rows and reads as a node with no run in this process; what was + stored stays stored. + Args: input_stream: The input stream to process. observer: Optional execution observer for hooks. @@ -1896,11 +1940,17 @@ def execute( if self._function_pod is None: raise self._cannot_compute_error() - self._begin_run() - # A whole-input condition, not a row failure: the error policy does not apply. - require_same_data_context( - input_stream.data_context_key, self._stored_data_context_key - ) + try: + self._begin_run() + # A whole-input condition, not a row failure: the error policy does not + # apply. + require_same_data_context( + input_stream.data_context_key, self._stored_data_context_key + ) + except BaseException: + # A run that raised defines no rows, also when it was refused at once. + self.clear_cache() + raise node_label = self.label node_hash = self.content_hash().to_string() @@ -1910,23 +1960,27 @@ def execute( tag_schema = input_stream.output_schema(columns={"system_tags": True})[0] ctx_obs.on_node_start(node_label, node_hash, tag_schema=tag_schema) - # Collect upstream entries and resolve base_entry_ids (stable across recomputation) - upstream_entries: list[tuple[TagProtocol, DataProtocol, bytes]] = [ - (tag, data, self.compute_base_entry_id(tag, data)) - for tag, data in input_stream.iter_data() - ] - base_entry_ids = [eid for _, _, eid in upstream_entries] - - # Hot-load any already-computed results from DB into _cached_output_datas. - # get_cached_results() is called for its side effect (populating the - # in-memory cache); the returned dict is intentionally discarded here so - # that the per-data cache-hit check below uses _cached_output_datas - # directly — which includes None-output entries (function returned None) - # and prevents spurious recomputation of already-processed data. - self.get_cached_results(base_entry_ids=base_entry_ids) - output: list[tuple[TagProtocol, DataProtocol]] = [] try: + # Collect upstream entries and resolve base_entry_ids (stable across + # recomputation) + upstream_entries: list[tuple[TagProtocol, DataProtocol, bytes]] = [ + (tag, data, self.compute_base_entry_id(tag, data)) + for tag, data in input_stream.iter_data() + ] + base_entry_ids = [eid for _, _, eid in upstream_entries] + + # Hot-load any already-computed results from DB into _cached_output_datas. + # get_cached_results() is called for its side effect (populating the + # in-memory cache); the returned dict is intentionally discarded here so + # that the per-data cache-hit check below uses _cached_output_datas + # directly — which includes None-output entries (function returned None) + # and prevents spurious recomputation of already-processed data. + self.get_cached_results(base_entry_ids=base_entry_ids) + + # Base entry ids of the inputs that gave a row (a cache hit or a + # successful computation), one per input: the node's rows afterwards. + emitted_entry_ids: list[bytes] = [] for tag, data, base_entry_id in upstream_entries: ctx_obs.on_data_start(node_label, tag, data) @@ -1935,6 +1989,7 @@ def execute( ): tag_out, result = self._cached_output_datas[base_entry_id] ctx_obs.on_data_end(node_label, tag, data, result, cached=True) + emitted_entry_ids.append(base_entry_id) if result is not None: output.append((tag_out, result)) continue @@ -1982,6 +2037,7 @@ def execute( continue ctx_obs.on_data_end(node_label, tag, data, result, cached=False) + emitted_entry_ids.append(base_entry_id) if result is not None: output.append((tag_out, result)) self._commit_or_end_node( @@ -1991,11 +2047,18 @@ def execute( self._commit_or_end_node( self._commit_at_node_end, ctx_obs, node_label, node_hash ) + # The node's rows are those of this run's inputs, whatever else it caches. + self._set_run_rows(emitted_entry_ids) + ctx_obs.on_node_end(node_label, node_hash) except Exception as run_error: self._commit_after_failure(run_error) + # A run that raised defines no rows: the node reads as one with no run. + self.clear_cache() + raise + except BaseException: + self.clear_cache() raise - ctx_obs.on_node_end(node_label, node_hash) # Mark this node as freshly computed so subsequent iter_data() calls # skip the is_stale check and serve results directly from the in-memory cache. self._update_modified_time() @@ -3086,8 +3149,8 @@ def head( ) -> StreamView: """Return the first rows the node holds or has stored; never runs the function. - After a run (or ``execute``) the rows are the node's results in memory, the - first of what ``as_table`` returns. Otherwise they are the first of its stored + After a run (or ``execute``) the rows are the node's results in memory for that + run's inputs, the first of what ``as_table`` returns. Otherwise they are the first of its stored rows: the node's whole stored slice is read and nothing is kept on the node. With nothing stored, there are no rows. The node has all of its rows, so with ``sort_by_tags`` (or ``all_info``) the head is exactly @@ -3107,9 +3170,11 @@ def head( config, ViewStatus.UNAVAILABLE, note="no store attached" ) current = status == LoadStatus.CACHE_ONLY or not self.is_stale - if current and self._cached_output_datas: + if current and ( + self._run_entry_counts is not None or self._cached_output_datas + ): return self._head_of_pairs( - list(self._cached_output_datas.values()), n, config, origin="held" + list(self._held_pairs()), n, config, origin="held" ) return self._head_from_store(n, config) @@ -3197,13 +3262,17 @@ def as_table( return super().as_table(columns=columns, all_info=all_info) def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: - """Yield all computed (tag, data) pairs for this node. + """Yield this node's (tag, data) rows. Strictly read-only — never triggers computation. Callers must call ``run()`` or ``execute()`` first if they want results computed. - On the first call with an empty in-memory store and a DB attached, - hot-loads all existing records from the DB (one-shot, no recompute). + After a completed run (``execute``, ``async_execute``, ``run()`` or a job run) + the rows are exactly those of that run's inputs, on either orchestrator, even + though results of other inputs stay cached and stored (ITL-1093). A run that + raised defines no rows. Without a completed run in this process, on the first + call with an empty in-memory store and a DB attached, hot-loads every record + stored under the node's content hash (one-shot, no recompute) and yields them. Raises: RuntimeError: If ``load_status`` is UNAVAILABLE. @@ -3222,11 +3291,7 @@ def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: if loaded: self._cached_output_table = None self._cached_content_hash_column = None - yield from ( - (tag, pkt) - for tag, pkt in self._cached_output_datas.values() - if pkt is not None - ) + yield from self._held_pairs() return # FULL / READ_ONLY — in-memory store may be populated from computation @@ -3245,11 +3310,7 @@ def iter_data(self) -> Iterator[tuple[TagProtocol, DataProtocol]]: self._cached_output_table = None self._cached_content_hash_column = None - yield from ( - (tag, pkt) - for tag, pkt in self._cached_output_datas.values() - if pkt is not None - ) + yield from self._held_pairs() def run(self) -> StreamView: """Eagerly compute all input data, filling pipeline and result databases, and @@ -3312,6 +3373,11 @@ async def async_execute( are served directly from persistent storage without touching the input channel. + Afterwards the node's rows are those of this run's inputs, as after + ``execute``, although every stored result was loaded into its cache. If the + run raises or is cancelled, it defines no rows: the node drops its in-memory + rows and reads as a node with no run in this process. + Args: input_channel: Single readable channel of (tag, data) pairs. output: Writable channel for output (tag, data) pairs. @@ -3329,7 +3395,9 @@ async def async_execute( try: self._begin_run() except StoreError: - # Nothing was delivered; downstream must not wait for rows. + # Nothing was delivered; downstream must not wait for rows. A run that + # raised defines no rows. + self.clear_cache() await output.close() raise status = self.load_status @@ -3354,6 +3422,8 @@ async def async_execute( self._input_stream.data_context_key, self._stored_data_context_key ) except InputValidationError: + # A run that raised defines no rows, also when it was refused at once. + self.clear_cache() await output.close() raise node_label = self.label @@ -3405,6 +3475,9 @@ async def async_execute( # Local dict: correlation_key → (original_tag, original_input_data) input_store: dict[bytes, tuple[TagProtocol, DataProtocol, bytes]] = {} + # Base entry ids of this run's inputs that gave a row (a cache hit or a + # forwarded computed row), one per input (ITL-1093). + run_entry_ids: list[bytes] = [] async def route_inputs() -> None: """Stage 1: send cache hits to output; stamp misses for computation.""" @@ -3427,6 +3500,7 @@ async def route_inputs() -> None: ctx_obs.on_data_end( node_label, tag, data, cached_data, cached=True ) + run_entry_ids.append(base_entry_id) await output.send((cached_tag, cached_data)) else: correlation_key = uuid.uuid4().bytes @@ -3484,6 +3558,7 @@ async def record_and_forward() -> None: # Commits every commit_every_rows rows inside a job (ITL-919); # no await in between, so no other row interleaves. self._row_processed() + run_entry_ids.append(base_entry_id) await output.send((clean_tag, output_data)) # Capture outer self so _NodeLabelObserver methods can @@ -3589,6 +3664,9 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: tg.create_task(record_and_forward()) # Commit the last rows inside a job (ITL-919); a no-op otherwise. self._commit_at_node_end() + # The node's rows are those of this run's inputs, as the sync path's: + # the cache also holds every stored result loaded above. + self._set_run_rows(run_entry_ids) else: # No-DB path: delegate dispatch entirely to the pod. The pod closes # the writer it is given; that close waits for the finally below, so @@ -3607,6 +3685,18 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: self._update_modified_time() except Exception as run_error: self._commit_after_failure(run_error) + # A run that raised defines no rows: the node reads as one with no run, + # as on the sync path. + self.clear_cache() + raise + except BaseException: + # Cancelled part-way: no rows either. + self.clear_cache() raise finally: - await output.close() + try: + await output.close() + except BaseException: + # A close that raises or is cancelled fails the run: no rows. + self.clear_cache() + raise diff --git a/superpowers/plans/2026-11-13-itl-1093-job-node-rows-after-run.md b/superpowers/plans/2026-11-13-itl-1093-job-node-rows-after-run.md new file mode 100644 index 000000000..bc3428932 --- /dev/null +++ b/superpowers/plans/2026-11-13-itl-1093-job-node-rows-after-run.md @@ -0,0 +1,87 @@ +# ITL-1093: a function job node returns the rows of its run's inputs — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** After a run on either orchestrator, a `FunctionJobNode` returns exactly the rows of +that run's inputs (`as_table`, `iter_data`, `head`, `run()`, a consumer built over it), row for +row equal between sync and async. Nothing stored changes. + +**Architecture:** `FunctionJobNode._cached_output_datas` stays a superset cache keyed by base +entry id. A new `_run_entry_ids: list[bytes] | None` records the base entry ids of the last +completed run's inputs in input order (`None` = no completed run in this process). One helper +`_held_pairs()` is the only reader of the held rows: with run ids it yields one row per run +input (not deduplicated, so held rows equal what downstream received), else every held value +(unchanged no-run behaviour). A run that raises defines no rows: the node clears its in-memory +rows and run marker (`clear_cache()`) and reads as a no-run node. + +**Tech Stack:** Python 3.11+, PyArrow, pytest, `uv run`. + +Coordinator decisions (November 13, 2026): proposal approved; no-run read unchanged (option A, +documented; B/C reported to the developer); duplicates not deduplicated; a raised run → no-run +read (option i); REPLAY union and async `fail_fast` (ITL-879) reported only. + +--- + +## Cause (measured on main b4fde330) + +- `async_execute` loads `self._load_cached_entries()` with no ids (every pipeline record under + the node's content hash) into `_cached_output_datas`; every read yields every dict value. +- Sync `execute` loads only this run's ids (`get_cached_results(base_entry_ids)`), so it looked + right only because a new job builds fresh node objects. Re-running the same job object with a + rebound source accumulates on sync too (3/6/6/9). + +## Files + +- Modify: `src/orcapod/core/nodes/function_node.py` (FunctionJobNode: `__init__`, `clear_cache`, + `execute`, `async_execute`, `iter_data`, `head`, new `_held_pairs`) +- Create: `tests/test_pipeline/test_job_node_rows_after_run.py` +- Create: `changelog.d/ITL-1093.md`, `design_issues/ITL-1093-JN1.md` +- Modify: `CLAUDE.md`, `.zed/rules` (Phase 1 sentence); `docs/concepts/` sentence if one exists. + +## Tasks + +### Task 1: Failing tests + +- [ ] Write `tests/test_pipeline/test_job_node_rows_after_run.py`: pipeline `rowwise` (inc over + DictSource source_id "s"), `downstream` (dbl over rowwise), `grouped` (total over + `src.group_by("g")`). Sequence 3 / 6 / 3 again / 3 different rows. + - fresh job per run, sync and async: each node's `as_table`, `iter_data`, `head(100)`, + `run()` and a consumer `dbl.pod(job.nodes["rowwise"])` return the rows of the run's inputs; + sync == async as sets of rows with system tags; the 6-row run reports 3 reused. + - same job object, `job.bind(sources=...)` per run, sync and async: same expectations. + - filtered inputs (function returns None) are in no read. + - a raised run (sync fail_fast ValueError; sync and async StoreWriteError raised part-way) + reads as a no-run node: equal to a fresh compiled job's read over the same store. +- [ ] Run (Python 3.12 and 3.11): `uv run pytest tests/test_pipeline/test_job_node_rows_after_run.py` + — expect async and same-job cases to FAIL. +- [ ] WIP commit. + +### Task 2: Fix + +- [ ] `_run_entry_ids` class default `None`, set in `__init__`; `clear_cache()` resets it. +- [ ] `_held_pairs()` helper; `iter_data` (both branches) and `head`'s held branch read through it; + `head` takes the held branch when `_run_entry_ids is not None` even with no held rows. +- [ ] `execute`: completed flag; at success set `_run_entry_ids = base_entry_ids` and drop the + cached table/hash column; on any raise after processing began, `_commit_after_failure` + (unchanged) then `clear_cache()`. +- [ ] `async_execute` DB path: `route_inputs` appends each input's base entry id to a local + list; at success set `_run_entry_ids`; a raise (Exception or cancellation) → `clear_cache()`. +- [ ] Run new tests (3.12 + 3.11) and targeted node/pipeline tests; commit. + +### Task 3: Docs and records + +- [ ] Docstrings of `iter_data`, `as_table`, `head`, `run`, `execute`, `async_execute` state the rule. +- [ ] `changelog.d/ITL-1093.md` under `### Fixed`; check with + `uv run python scripts/assemble_changelog.py --check`. +- [ ] `design_issues/ITL-1093-JN1.md` (resolved, Fix note); `uv run python scripts/design_issues.py --check`. +- [ ] CLAUDE.md + `.zed/rules` sentence → "After a run, a `FunctionJobNode` returns exactly the rows + of that run's inputs, sync or async (a run that raised defines no rows). Read without a + completed run in this process (fresh or loaded job), it returns every record stored under its + content hash for the current data context. A run skips inputs whose entry is already stored." +- [ ] docs/concepts sentence if one states what a job node returns after a run. +- [ ] ruff, ruff format, basedpyright (exit code), golden suite, targeted suites; commit; push; draft PR. + +## Notes for a successor + +- Probes: scratchpad (not durable); measured tables are in the coordinator ask and the PR body. +- PR #359 (ITL-1063) edits the same `async_execute`/`execute` hunks; semantics compatible. diff --git a/tests/test_pipeline/test_job_node_rows_after_run.py b/tests/test_pipeline/test_job_node_rows_after_run.py new file mode 100644 index 000000000..c31e7c487 --- /dev/null +++ b/tests/test_pipeline/test_job_node_rows_after_run.py @@ -0,0 +1,540 @@ +"""After a run, a function job node returns exactly the rows of that run's inputs (ITL-1093). + +One store, a source of one ``source_id`` holding 3 rows, then 6, then the same 3 again, then +3 different rows. After each run, on either orchestrator, every function job node returns +the rows of that run's inputs through ``as_table``, ``iter_data``, ``head`` and ``run()``, +and so does a consumer built over the node afterwards; the two orchestrators agree row for +row. Stored results of other inputs stay stored and reusable. A run that raised defines no +rows: the node then reads as a node with no run in this process (every row stored under +its content hash). +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.core.data_function import PythonDataFunction +from orcapod.databases import InMemoryArrowDatabase +from orcapod.errors import StoreWriteError +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.sources import DictSource + + +@function_pod(output_keys="y") +def inc(x: int) -> int: + """Return ``x + 1``.""" + return x + 1 + + +@function_pod(output_keys="z") +def dbl(y: int) -> int: + """Return ``2 * y``.""" + return 2 * y + + +@function_pod(output_keys="total") +def total(x: list[int]) -> int: + """Return the sum of a group's values.""" + return sum(x) + + +@function_pod(output_keys="y") +def null_on_10(x: int) -> int | None: + """Return ``x + 1``, or a null value for ``x == 10``.""" + return None if x == 10 else x + 1 + + +@function_pod(output_keys="y") +def fails_on_20(x: int) -> int: + """Return ``x + 1``; raise for ``x == 20``.""" + if x == 20: + raise ValueError("x is 20") + return x + 1 + + +@function_pod(output_keys="y") +def store_fails_on_20(x: int) -> int: + """Return ``x + 1``; for ``x == 20`` raise the error a failed store write raises. + + A ``StoreError`` fails the run whatever the error policy, on both orchestrators, as a + failed write of the node's results does. + """ + if x == 20: + raise StoreWriteError(("simulated", "results")) + return x + 1 + + +ROWS = [{"g": i % 2, "k": i, "x": 10 * i} for i in range(6)] +# 3 rows, 6 rows (3 more), the same 3 again (fewer), 3 different rows. The source's record +# ids are positional, so the last run's rows are new inputs whose results are stored. +SEQUENCE = [ROWS[:3], ROWS, ROWS[:3], ROWS[3:]] +ORCHESTRATORS = ["sync", "async"] + + +def _source(data: list[dict[str, int]]) -> DictSource: + """Return a ``DictSource`` of ``data`` with tags ``g`` and ``k`` and source id ``s``.""" + return DictSource(data, tag_columns=["g", "k"], source_id="s") + + +def _build(data: list[dict[str, int]], db: Any) -> PipelineJob: + """Return a job over ``db``: ``rowwise``, ``downstream`` of it, and ``grouped``.""" + job = PipelineJob(name="itl1093", database=db) + with job: + rowwise = inc.pod(_source(data), label="rowwise") + dbl.pod(rowwise, label="downstream") + total.pod(_source(data).group_by("g"), label="grouped") + return job + + +def _orchestrator(kind: str, error_policy: str = "continue") -> Any: + """Return the ``sync`` or ``async`` orchestrator with ``error_policy``.""" + if kind == "sync": + return SyncPipelineOrchestrator(error_policy=error_policy) # pyright: ignore[reportArgumentType] + return AsyncPipelineOrchestrator(error_policy=error_policy) # pyright: ignore[reportArgumentType] + + +def _canonical(rows: list[dict[str, Any]]) -> list[tuple[tuple[str, str], ...]]: + """Rows as a sorted list, each a sorted tuple of (column, repr(value)).""" + return sorted(tuple(sorted((k, repr(v)) for k, v in row.items())) for row in rows) + + +ALL = {"system_tags": True} + + +def _reads(node: Any) -> dict[str, list[tuple[tuple[str, str], ...]]]: + """Every public read of ``node``, rows with system tags, canonical.""" + iterated = [ + {**tag.as_dict(all_info=True), **data.as_dict()} + for tag, data in node.iter_data() + ] + return { + "as_table": _canonical(node.as_table(columns=ALL).to_pylist()), + "head": _canonical(node.head(100, columns=ALL).as_table().to_pylist()), + "iter_data": _canonical( + [ + {k: v for k, v in row.items() if not k.startswith("__")} + for row in iterated + ] + ), + } + + +def _values(node: Any, key: Callable[[dict[str, Any]], Any]) -> list[Any]: + """Return ``key`` of every row of ``node.as_table()``, sorted.""" + return sorted(key(row) for row in node.as_table().to_pylist()) + + +def _expected_rowwise(data: list[dict[str, int]]) -> list[tuple[int, int]]: + """Return the ``(k, y)`` rows ``rowwise`` gives for ``data``, sorted.""" + return sorted((row["k"], row["x"] + 1) for row in data) + + +def _expected_downstream(data: list[dict[str, int]]) -> list[tuple[int, int]]: + """Return the ``(k, z)`` rows ``downstream`` gives for ``data``, sorted.""" + return sorted((row["k"], 2 * (row["x"] + 1)) for row in data) + + +def _expected_grouped(data: list[dict[str, int]]) -> list[tuple[int, int]]: + """Return the ``(g, total)`` rows ``grouped`` gives for ``data``, sorted.""" + sums: dict[int, int] = {} + for row in data: + sums[row["g"]] = sums.get(row["g"], 0) + row["x"] + return sorted(sums.items()) + + +def _check_run(job: PipelineJob, data: list[dict[str, int]]) -> None: + """Assert every node of ``job`` returns the rows of ``data``, through every read.""" + nodes = job.nodes + assert _values(nodes["rowwise"], lambda r: (r["k"], r["y"])) == _expected_rowwise( + data + ) + assert _values(nodes["downstream"], lambda r: (r["k"], r["z"])) == ( + _expected_downstream(data) + ) + assert _values(nodes["grouped"], lambda r: (r["g"], r["total"])) == ( + _expected_grouped(data) + ) + for label in ("rowwise", "downstream", "grouped"): + reads = _reads(nodes[label]) + assert reads["head"] == reads["as_table"], label + assert len(reads["iter_data"]) == len(reads["as_table"]), label + # A consumer built over the node after the run receives the run's rows. + consumer = dbl.pod(nodes["rowwise"]) + assert sorted((r["k"], r["z"]) for r in consumer.as_table().to_pylist()) == ( + _expected_downstream(data) + ) + + +def _snapshot(job: PipelineJob) -> dict[str, Any]: + """Return every public read of each function node of ``job``.""" + return { + label: _reads(job.nodes[label]) + for label in ("rowwise", "downstream", "grouped") + } + + +class TestFreshJobPerRun: + """A new job over the same store for every run, as the issue's reproduction.""" + + @pytest.mark.parametrize("kind", ORCHESTRATORS) + def test_each_run_returns_its_inputs_rows(self, kind: str) -> None: + """Every read after each run gives that run's rows; the full run reuses 3.""" + db = InMemoryArrowDatabase() + for index, data in enumerate(SEQUENCE): + job = _build(data, db) + summary = job.run(orchestrator=_orchestrator(kind)) + _check_run(job, data) + if index == 1: + # The 6-row run reuses the 3 stored results of the first run. + assert summary.nodes["rowwise"].reused == 3 + assert summary.nodes["rowwise"].computed == 3 + + def test_sync_and_async_agree_row_for_row(self) -> None: + """Both orchestrators give equal rows, system tags included, after every run.""" + snapshots: dict[str, list[dict[str, Any]]] = {} + for kind in ORCHESTRATORS: + db = InMemoryArrowDatabase() + snapshots[kind] = [] + for data in SEQUENCE: + job = _build(data, db) + job.run(orchestrator=_orchestrator(kind)) + snapshots[kind].append(_snapshot(job)) + assert snapshots["sync"] == snapshots["async"] + + @pytest.mark.parametrize("kind", ORCHESTRATORS) + def test_node_run_returns_its_inputs_rows(self, kind: str) -> None: + """A node's ``run()`` after the job run shows that run's rows.""" + db = InMemoryArrowDatabase() + for data in SEQUENCE: + job = _build(data, db) + job.run(orchestrator=_orchestrator(kind)) + view = job.nodes["rowwise"].run() + assert sorted((r["k"], r["y"]) for r in view.as_table().to_pylist()) == ( + _expected_rowwise(data) + ) + + +class TestSameJobRebound: + """One job object, its source rebound before every run: the same node objects run again.""" + + @pytest.mark.parametrize("kind", ORCHESTRATORS) + def test_each_run_returns_its_inputs_rows(self, kind: str) -> None: + """Every read after each run of the rebound job gives that run's rows.""" + db = InMemoryArrowDatabase() + job = _build(SEQUENCE[0], db) + for data in SEQUENCE: + job.bind(sources={"s": _source(data)}) + job.run(orchestrator=_orchestrator(kind)) + _check_run(job, data) + + def test_sync_and_async_agree_row_for_row(self) -> None: + """Both orchestrators give equal rows after every run of the rebound job.""" + snapshots: dict[str, list[dict[str, Any]]] = {} + for kind in ORCHESTRATORS: + db = InMemoryArrowDatabase() + job = _build(SEQUENCE[0], db) + snapshots[kind] = [] + for data in SEQUENCE: + job.bind(sources={"s": _source(data)}) + job.run(orchestrator=_orchestrator(kind)) + snapshots[kind].append(_snapshot(job)) + assert snapshots["sync"] == snapshots["async"] + + +def _single(pod: Any, data: list[dict[str, int]], db: Any) -> PipelineJob: + """Return a job over ``db`` applying ``pod`` to a source of ``data`` as ``node``.""" + job = PipelineJob(name="itl1093_single", database=db) + with job: + pod(_source(data), label="node") + return job + + +class TestNullAndInactive: + """A null value is a row; an inactive function's inputs (no output) are rows of no read.""" + + @pytest.mark.parametrize("kind", ORCHESTRATORS) + def test_null_value_is_a_row(self, kind: str) -> None: + """A row whose output value is null is one of the run's rows.""" + db = InMemoryArrowDatabase() + _single(null_on_10.pod, ROWS, db).run(orchestrator=_orchestrator(kind)) + job = _single(null_on_10.pod, ROWS[:3], db) + job.run(orchestrator=_orchestrator(kind)) + node = job.nodes["node"] + assert _values(node, lambda r: (r["k"], r["y"])) == [(0, 1), (1, None), (2, 21)] + reads = _reads(node) + assert reads["head"] == reads["as_table"] + assert len(reads["iter_data"]) == 3 + + @pytest.mark.parametrize("kind", ORCHESTRATORS) + def test_inactive_function_gives_no_rows(self, kind: str) -> None: + """A run of an inactive function has no rows, though other results are stored.""" + + @function_pod(output_keys="y") + def add_one(x: int) -> int: + """Return ``x + 1``.""" + return x + 1 + + db = InMemoryArrowDatabase() + _single(add_one.pod, ROWS[3:], db).run(orchestrator=_orchestrator(kind)) + # A function switched off produces no output for the inputs it is given. + data_function: object = add_one.pod.data_function + assert isinstance(data_function, PythonDataFunction) + data_function.set_active(False) + job = _single(add_one.pod, ROWS[:3], db) + job.run(orchestrator=_orchestrator(kind)) + node = job.nodes["node"] + assert node.as_table().num_rows == 0 + assert list(node.iter_data()) == [] + assert node.head(100).as_table().num_rows == 0 + + +class TestRaisedRun: + """A run that raised defines no rows: the node reads as a node with no run.""" + + @staticmethod + def _no_run_read(fn: Any, data: list[dict[str, int]], db: Any) -> list[Any]: + """Return the ``(k, y)`` rows a compiled, unrun job over ``db`` reads.""" + fresh = _single(fn.pod, data, db) + fresh.compile() + return _values(fresh.nodes["node"], lambda r: (r["k"], r["y"])) + + @pytest.mark.parametrize( + ("kind", "fn", "policy"), + [ + ("sync", fails_on_20, "fail_fast"), + ("sync", store_fails_on_20, "continue"), + ("async", store_fails_on_20, "continue"), + ], + ) + def test_raised_run_reads_as_no_run(self, kind: str, fn: Any, policy: str) -> None: + """After a run that raised, the node reads every stored row, as with no run.""" + db = InMemoryArrowDatabase() + # Store the results of rows 3..5 first, then run rows 0..3, which fails at x=20. + _single(fn.pod, ROWS[3:], db).run(orchestrator=_orchestrator(kind)) + job = _single(fn.pod, ROWS[:4], db) + with pytest.raises((ValueError, StoreWriteError)): + job.run(orchestrator=_orchestrator(kind, policy)) + node = job.nodes["node"] + expected = self._no_run_read(fn, ROWS[:4], db) + assert _values(node, lambda r: (r["k"], r["y"])) == expected + reads = _reads(node) + assert reads["head"] == reads["as_table"] + # Nothing stored was dropped: the stored rows of the earlier run are among them. + assert {(3, 31), (4, 41), (5, 51)} <= set(expected) + + +class TestDuplicateInputs: + """Two inputs with one base entry id give two held rows, as downstream receives them. + + Only a stream without system tags can carry two such inputs (a source gives every + row its own record id), so the node is built directly over an ``ArrowTableStream``. + """ + + @staticmethod + def _node(pod: Any) -> Any: + """Return a node applying ``pod`` to rows (k=0, x=5) twice and (k=2, x=7).""" + import pyarrow as pa + + from orcapod.core.nodes import FunctionJobNode + from orcapod.core.streams import ArrowTableStream + + schema = pa.schema( + [ + pa.field("k", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] + ) + table = pa.Table.from_arrays( + [pa.array([0, 0, 2], pa.int64()), pa.array([5, 5, 7], pa.int64())], + schema=schema, + ) + return FunctionJobNode( + pod, + ArrowTableStream(table, tag_columns=["k"]), + pipeline_database=InMemoryArrowDatabase(), + result_database=InMemoryArrowDatabase(), + ) + + @staticmethod + def _flaky_pod() -> Any: + """Return a pod computing ``x + 1`` whose first call for ``x == 5`` raises.""" + calls: list[int] = [] + + @function_pod(output_keys="y") + def flaky_inc(x: int) -> int: + """Return ``x + 1``; raise on the first call for ``x == 5``.""" + calls.append(x) + if x == 5 and calls.count(5) == 1: + raise ValueError("transient failure") + return x + 1 + + return flaky_inc.pod + + @staticmethod + def _run_async(node: Any) -> list[tuple[int, int]]: + """Run ``node`` with ``async_execute`` over its upstream; return the rows sent.""" + import asyncio + + from orcapod.channels import Channel + + async def run() -> list[tuple[int, int]]: + """Feed the upstream's rows in and collect what the node sends out.""" + inputs = Channel(buffer_size=16) + outputs = Channel(buffer_size=16) + # The input stream is the node's own upstream (a test of async_execute). + for item in node.upstreams[0].iter_data(): + await inputs.writer.send(item) + await inputs.writer.close() + await node.async_execute(inputs.reader, outputs.writer) + got = [ + (tag.as_dict()["k"], data.as_dict()["y"]) + async for tag, data in outputs.reader + ] + return sorted(got) + + return asyncio.run(run()) + + @staticmethod + def _held(node: Any) -> list[tuple[int, int]]: + """Return the node's ``(k, y)`` rows, sorted.""" + return sorted((r["k"], r["y"]) for r in node.as_table().to_pylist()) + + def test_sync(self) -> None: + """Sync: both equal inputs are held, as the run returned them.""" + node = self._node(inc.pod) + view = node.run() + assert self._held(node) == [(0, 6), (0, 6), (2, 8)] + assert sorted((r["k"], r["y"]) for r in view.as_table().to_pylist()) == ( + self._held(node) + ) + + def test_async(self) -> None: + """Async: both equal inputs are held, as they were sent downstream.""" + node = self._node(inc.pod) + received = self._run_async(node) + assert received == [(0, 6), (0, 6), (2, 8)] + assert self._held(node) == received + + def test_sync_one_of_two_equal_inputs_fails(self) -> None: + """Sync: an equal input that failed is not held; the one that succeeded is.""" + node = self._node(self._flaky_pod()) + received = sorted( + (tag.as_dict()["k"], data.as_dict()["y"]) + for tag, data in node.execute(node.upstreams[0]) + ) + assert received == [(0, 6), (2, 8)] + assert self._held(node) == received + + def test_async_one_of_two_equal_inputs_fails(self) -> None: + """Async: an equal input that failed is not held; the one that succeeded is.""" + node = self._node(self._flaky_pod()) + received = self._run_async(node) + assert received == [(0, 6), (2, 8)] + assert self._held(node) == received + + +class TestRefusedOrFailedRunAfterAGoodRun: + """A run that raises after a good run drops that run's rows too (ITL-1093). + + The node then reads as a node with no run in this process: every row stored under + its content hash, read from the store, whatever raised. An input refused for its data + context is tested in ``test_stored_data_context.py``, which registers a second one. + """ + + @staticmethod + def _job(db: Any) -> PipelineJob: + """Return a job applying ``inc`` to a source of 6 rows as ``node``.""" + return _single(inc.pod, ROWS, db) + + @pytest.mark.parametrize("kind", ORCHESTRATORS) + def test_observer_failure_at_node_end_drops_the_rows(self, kind: str) -> None: + """An observer raising at the node's end leaves the node with no run's rows.""" + from orcapod.pipeline.observer import NoOpObserver + + class FailsAtNodeEnd(NoOpObserver): + """An observer whose ``on_node_end`` raises.""" + + def on_node_end(self, node_label: str, node_hash: str) -> None: + """Raise at the function node's end, as a broken observer would.""" + if node_label == "node": + raise RuntimeError("observer failed") + + db = InMemoryArrowDatabase() + job = self._job(db) + job.run(orchestrator=_orchestrator(kind)) + job.bind(sources={"s": _source(ROWS[:3])}) + job.run(orchestrator=_orchestrator(kind)) + node = job.nodes["node"] + assert _values(node, lambda r: r["k"]) == [0, 1, 2] + # The async orchestrator raises its task group's error around it. + with pytest.raises((RuntimeError, ExceptionGroup)): + job.run(orchestrator=_orchestrator(kind), observer=FailsAtNodeEnd()) + # Every stored row (the 6-row run's), read from the store. + assert _values(node, lambda r: r["k"]) == [0, 1, 2, 3, 4, 5] + + +class TestAsyncOutputCloseFails: + """An async run whose output cannot be closed defines no rows (ITL-1093).""" + + def test_failed_close_leaves_no_run_rows(self) -> None: + """A close that raises after every row was sent leaves the node with no run.""" + import asyncio + + import pyarrow as pa + + from orcapod.channels import Channel + from orcapod.core.nodes import FunctionJobNode + from orcapod.core.streams import ArrowTableStream + + class CloseFails: + """An output writer that keeps what it is sent and raises on close.""" + + def __init__(self) -> None: + """Start with no rows.""" + self.sent: list[Any] = [] + + async def send(self, item: Any) -> None: + """Keep ``item``.""" + self.sent.append(item) + + async def close(self) -> None: + """Raise, as a closed or broken channel would.""" + raise OSError("output cannot be closed") + + schema = pa.schema( + [ + pa.field("k", pa.int64(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] + ) + table = pa.Table.from_arrays( + [pa.array([0, 1], pa.int64()), pa.array([5, 6], pa.int64())], + schema=schema, + ) + node = FunctionJobNode( + inc.pod, # pyright: ignore[reportArgumentType] # the decorator's pod type is not declared as FunctionPodProtocol + ArrowTableStream(table, tag_columns=["k"]), + pipeline_database=InMemoryArrowDatabase(), + result_database=InMemoryArrowDatabase(), + ) + output = CloseFails() + + async def run() -> None: + """Run the node over its upstream's rows into ``output``.""" + inputs = Channel(buffer_size=16) + # The input stream is the node's own upstream (a test of async_execute). + for item in node.upstreams[0].iter_data(): + await inputs.writer.send(item) + await inputs.writer.close() + await node.async_execute(inputs.reader, output) # pyright: ignore[reportArgumentType] # a test double of the writer + + with pytest.raises(OSError, match="cannot be closed"): + asyncio.run(run()) + assert len(output.sent) == 2 + # No run is held: the rows are read from the store. + assert node.head(10).origin == "stored" + assert _values(node, lambda r: (r["k"], r["y"])) == [(0, 6), (1, 7)] diff --git a/tests/test_pipeline/test_stored_data_context.py b/tests/test_pipeline/test_stored_data_context.py index 49708925a..17f072b73 100644 --- a/tests/test_pipeline/test_stored_data_context.py +++ b/tests/test_pipeline/test_stored_data_context.py @@ -1254,6 +1254,38 @@ def test_function_node_raises_and_writes_nothing(self, store: Store): assert calls == [] assert reopened(store).list_sources() == [] + def test_refused_input_after_a_good_run_leaves_no_run_rows(self, store: Store): + """A run refused for its input's data context defines no rows (ITL-1093). + + After a good run the node holds that run's rows; a later sync run refused for its + input's data context drops them, and the node reads its stored rows, as with no + run in this process. (On the async path the context checked is the node's own + input stream's, so a node refused there never had a run.) + """ + table = pa.table( + {"k": ["a", "b"], "x": [1, 2]}, + schema=pa.schema( + [ + pa.field("k", pa.large_string(), nullable=False), + pa.field("x", pa.int64(), nullable=False), + ] + ), + ) + node = FunctionJobNode( + function_pod=double.pod, # pyright: ignore[reportArgumentType] # the decorator types .pod as the pod base class + input_stream=ArrowTableStream(table, tag_columns=["k"]), + pipeline_database=store, + result_database=store, + ) + node.run() + assert node.head(10).origin == "held" + + with pytest.raises(InputValidationError): + node.execute(_other_context_stream()) + + assert node.head(10).origin == "stored" + assert sorted(row["double"] for row in node.as_table().to_pylist()) == [2, 4] + def test_add_pipeline_record_refuses_an_input_of_another_data_context( self, store: Store ):