From 3219f5babafaeda8066ebdf0cd6ec5fd2faa0a3f Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:05:37 -0800 Subject: [PATCH 1/7] test(async_orchestrator): failing tests for error_policy on the async path (ITL-879) Co-Authored-By: Claude Opus 5.5 --- .../2026-11-13-itl-879-async-error-policy.md | 160 +++++++ .../test_pipeline/test_async_error_policy.py | 452 ++++++++++++++++++ 2 files changed, 612 insertions(+) create mode 100644 superpowers/plans/2026-11-13-itl-879-async-error-policy.md create mode 100644 tests/test_pipeline/test_async_error_policy.py diff --git a/superpowers/plans/2026-11-13-itl-879-async-error-policy.md b/superpowers/plans/2026-11-13-itl-879-async-error-policy.md new file mode 100644 index 000000000..2fa0f8bad --- /dev/null +++ b/superpowers/plans/2026-11-13-itl-879-async-error-policy.md @@ -0,0 +1,160 @@ +# ITL-879 Async error_policy Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:subagent-driven-development (recommended) or sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** `error_policy` means the same on `AsyncPipelineOrchestrator` as on `SyncPipelineOrchestrator`: `fail_fast` raises the failing row's own exception (same type and message as sync), `continue` skips and reports the row as it already does; an async run never raises an `ExceptionGroup` to its caller. + +**Architecture:** Option A (coordinator-approved, November 13, 2026): a keyword-only +`error_policy: Literal["continue", "fail_fast"] = "continue"` on +`FunctionNodeProtocol.async_execute`, `FunctionJobNode.async_execute` and +`_FunctionPodBase.async_execute`, mirroring `execute`. The pod's per-row handler re-raises a +row failure under `fail_fast`; the existing task groups cancel the rest and every node closes +its output in its existing `finally`. The orchestrator's `except ExceptionGroup` handler picks +ONE exception: store errors, then `TypeIdentityError` (both unchanged), then the failure of +the node earliest in topological order (ties: group order = completion order); the others are +added to it as notes. The orchestrator's cancel/close machinery is not changed. + +**Tech Stack:** Python 3.11+, asyncio `TaskGroup`, pytest. + +## Measured on main b4fde330 (before) + +6 rows, row r2 raises `ValueError("bad x=2")`, function node `f` -> function node `down`. + +| | sync continue | async continue | sync fail_fast | async fail_fast | +|---|---|---|---|---| +| raises | no | no | `ValueError: bad x=2` (no cause, no notes) | **no** | +| RunSummary f | 5 computed, 1 failed | same | none (last_run unchanged) | **5 computed, 1 failed** | +| RowFailuresWarning | 1 | 1 | none | **1** | +| per-row log | DEBUG function_node | DEBUG function_pod | DEBUG | DEBUG | +| observer | crash r2, end x5, node_end f, down | same (interleaved) | end r0 r1, crash r2, node_end f, run_end; down never starts | **as continue** | +| stored / held f | r0 r1 r3 r4 r5 | same | r0 r1 | **r0 r1 r3 r4 r5** | +| stored / held down | r0 r1 r3 r4 r5 | same | nothing | **r0 r1 r3 r4 r5** | + +SideEffectPod (own `on_error`, not `error_policy`): `on_error="raise"`: sync raises bare +`ValueError`, async raises `ExceptionGroup([ValueError])`. `on_error="log"`: identical +(WARNING log, other rows pass). + +## Storage guarantee (approved wording) + +"After a fail_fast raise, every stored row is a correct, complete result of its own input +row, written in the ITL-878 commit order (results before pipeline records; writes that +succeeded are flushed before the raise). The failing row is never stored. Other rows of the +failing node, before or after it in input order, rows that downstream nodes computed from its +successful rows, and rows of other branches may be stored; the set stored is not specified +beyond that. When the cancellation falls between a row's result write and its pipeline-record +write, the result is stored without its record: a re-run reuses it without calling the +function, as after a crash." + +## Files + +- Modify `src/orcapod/protocols/node_protocols.py` (`FunctionNodeProtocol.async_execute`). +- Modify `src/orcapod/core/function_pod.py` (`_FunctionPodBase.async_execute`). +- Modify `src/orcapod/core/nodes/function_node.py` (`FunctionJobNode.async_execute`). +- Modify `src/orcapod/pipeline/async_orchestrator.py` (pass the policy; `_start_node` records + which node raised; `_single_failure` picks the exception). +- Modify `src/orcapod/utils/store_errors.py` (`store_error_of` helper for the node's + `on_node_end` on failure). +- Create `tests/test_pipeline/test_async_error_policy.py`. +- Docs: `docs/concepts/function-pods.md`, `README.md` diagram if it states async behaviour. +- `changelog.d/ITL-879.md`, `design_issues/ITL-878-SW1.md` note, new + `design_issues/ITL-879-EP1.md` entry (resolved). + +## Tasks + +### Task 1: failing tests (TDD) + +`tests/test_pipeline/test_async_error_policy.py`, public interface only (PipelineJob.run / +run_async), except the channel-closed check (patched `Channel`/`BroadcastChannel` in +`orcapod.pipeline.async_orchestrator`, private `_closed` read; commented). + +- `test_continue_parity[sync|async]` x scenarios fn / sef: no raise, RunSummary counts, + one RowFailuresWarning, observer crash event, rows of f and down = all but the failing row. +- `test_fail_fast_raises_the_row_error[sync|async]` x fn / sef: `pytest.raises(ValueError)`, + `type(err) is ValueError`, `str(err) == "bad x=2"`, not an ExceptionGroup, `job.last_run is + None`, no RowFailuresWarning, observer got `on_data_crash` for the failing row and + `on_node_end` for f. +- `test_fail_fast_stored_rows_are_correct[async]`: every row a fresh job reads for f and down + is a correct result (y == x, z == 2x) and the failing row is absent. +- `test_downstream_never_completes_on_partial_input` parametrized failing row first / middle / + last x max_concurrency 1 / None: async fail_fast raises; `down` gets no `on_node_end`. +- `test_max_concurrency_one_raises_the_first_row_in_input_order`: two failing rows, first one + raised; the second is not computed (sync and async). +- `test_side_effect_pod_raise[sync|async]` x policies: bare `ValueError`, not a group. +- `test_side_effect_pod_log[sync|async]`: no raise, WARNING log, other rows pass. +- `test_nothing_left_running_after_fail_fast`: `await job.run_async(AsyncPipelineOrchestrator( + error_policy="fail_fast"))` inside `async with asyncio.timeout(60)`, then + `asyncio.all_tasks() - {asyncio.current_task()} == set()` and every channel created closed. +- `test_two_nodes_fail_raises_the_upstream_failure`: async, both f and a downstream fail_fast + node fail; raised is f's, the other appears in `__notes__` (only when it happened). + +Run: `uv run pytest tests/test_pipeline/test_async_error_policy.py -q` -> the async fail_fast +and side-effect-raise cases FAIL; continue and sync cases PASS. Commit (WIP test commit). + +### Task 2: policy through the async path + +1. `node_protocols.py`: add `error_policy: Literal["continue", "fail_fast"] = "continue"` to + `FunctionNodeProtocol.async_execute` (keyword-only, after `observer`). +2. `function_pod.py` `_FunctionPodBase.async_execute`: same keyword; docstring; in + `process_one`'s `except Exception` after `obs.on_data_crash(...)`: + ```python + if error_policy == "fail_fast": + raise + ``` +3. `function_node.py` `FunctionJobNode.async_execute`: same keyword; pass + `error_policy=error_policy` to `execution_pod.async_execute` and to the no-DB + `self._function_pod.async_execute`. Track `node_started = True` after `on_node_start`; in + `except Exception as run_error:` before `_commit_after_failure`: + ```python + if node_started: + # The sync execute ends the node for the observer before it raises. + with keep_store_error(store_error_of(run_error), "Observer on_node_end"): + ctx_obs.on_node_end(node_label, node_hash) + ``` +4. `store_errors.py`: `store_error_of(error)` returns the `StoreError`/`TypeIdentityError` + that is `error` or the first one inside it when it is an exception group, else `None`. +5. `async_orchestrator.py`: pass `error_policy=self._error_policy` to function and + side-effect-function nodes' `async_execute`. + +### Task 3: one exception from an async run + +`async_orchestrator.py`: +- `_start_node(tg, node, work, failed_nodes)`: on `except Exception as exc` record + `failed_nodes[id(leaf)] = node.label` for every leaf of `exc`, then re-raise (store-error + annotation unchanged). +- Handler: after the store-error and identity-error branches, + `raise _single_failure(group, topo_order, failed_nodes)`: the leaf whose node ranks first + in `topo_order`, ties by group order; each other leaf noted as + `f"Node {label!r} also failed in the same run: {type(other).__name__}: {other}"` (label + omitted when unknown). +- Docstrings of `run`/`run_async`: "Raises: the exception of the failing node itself, never an + exception group". + +Run the new tests -> PASS. Run targeted existing suites (one pytest process): +`tests/test_pipeline/ tests/test_core/function_pod/ tests/test_core/side_effect_function/ +tests/test_channels/` on 3.11 and 3.12 for the new file. Commit. + +### Task 4: docs, changelog, design issues + +- `docs/concepts/function-pods.md`: replace the two "async ... ITL-879" sentences; next to the + `fail_fast` paragraph state behaviour on both orchestrators, "first" under concurrency, and + the storage guarantee verbatim. Run `uv run pytest tests/test_docs -q -k function_pods`. +- `changelog.d/ITL-879.md`: `### Breaking changes and migration` (async fail_fast now raises; + async runs raise the node's exception, never an ExceptionGroup) and `### Fixed`. + `uv run python scripts/assemble_changelog.py --check`. +- `design_issues/ITL-878-SW1.md`: the "unchanged here" sentence gains "(fixed by ITL-879)". + New `design_issues/ITL-879-EP1.md` (resolved, with Fix note). + `uv run python scripts/design_issues.py --check`. +- ruff format/check, basedpyright exit code. + +### Task 5: finish + +Merge origin/main (after #378 if merged), rerun targeted tests, full suite once, push, draft +PR "Fixes ITL-879", CodeRabbit process per brief. + +## Notes for a successor + +- Proposal approved by coordinator (Option A, unwrap every node failure, guarantee wording). +- Merge order: PR #378 (ITL-1093) first; conflicts expected in `FunctionJobNode.async_execute` + except/finally region. Its rule: a run that raised leaves the node holding nothing. +- If the downstream-partial-input test ever fails: STOP, ask coordinator (would need + close-with-error channels). diff --git a/tests/test_pipeline/test_async_error_policy.py b/tests/test_pipeline/test_async_error_policy.py new file mode 100644 index 000000000..a4ce675ed --- /dev/null +++ b/tests/test_pipeline/test_async_error_policy.py @@ -0,0 +1,452 @@ +"""``error_policy`` means the same on both orchestrators (ITL-879). + +Before the fix the async orchestrator stored ``error_policy`` and never used it: under +``"fail_fast"`` a row whose function raised was skipped as under ``"continue"`` and the run +finished. A failing node's exception also reached the caller inside an ``ExceptionGroup``. +Now ``"fail_fast"`` raises the row's own exception, as the sync orchestrator does, and an +async run raises one exception, never a group. + +The jobs are driven through ``PipelineJob.run`` / ``run_async``. Every async run sits inside +an ``asyncio.timeout`` guard, so a hang fails the test instead of the suite. +""" + +from __future__ import annotations + +import asyncio +import logging +import warnings +from collections.abc import Callable, Iterator +from typing import Any + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.core.data_function import PythonDataFunction +from orcapod.core.function_pod import FunctionPod, side_effect_function_pod +from orcapod.databases import InMemoryArrowDatabase +from orcapod.errors import RowFailuresWarning +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.pipeline import async_orchestrator as async_orchestrator_module +from orcapod.pipeline.observer import NoOpObserver +from orcapod.side_effects import InvocationContext, SideEffectPodConfig, side_effect_pod +from orcapod.sources import DictSource +from orcapod.types import PodConfig + +RUN_TIMEOUT_SECONDS = 60 +N_ROWS = 6 + +ORCHESTRATORS: dict[str, type[Any]] = { + "sync": SyncPipelineOrchestrator, + "async": AsyncPipelineOrchestrator, +} + +# Where the failing row sits among the source rows. +FAILING_POSITIONS = {"first": 0, "middle": N_ROWS // 2, "last": N_ROWS - 1} + + +# --------------------------------------------------------------------------- +# Functions under test: a row fails when its x is negative +# --------------------------------------------------------------------------- + + +def check(x: int) -> int: + """Return ``x``; raise for a negative ``x``.""" + if x < 0: + raise ValueError(f"bad x={x}") + return x + + +checked = function_pod(output_keys="y")(check) + + +@function_pod(output_keys="z") +def double(y: int) -> int: + """Return twice ``y``.""" + return 2 * y + + +@side_effect_function_pod(output_keys="y") +def check_with_ctx(x: int, ctx: InvocationContext) -> int: + """Return ``x``; raise for a negative ``x`` (a pod that takes the invocation context).""" + if x < 0: + raise ValueError(f"bad x={x}") + return x + + +def deliver(x: int, ctx: InvocationContext) -> None: + """A side effect that fails for a negative ``x``.""" + if x < 0: + raise ValueError(f"bad x={x}") + + +def xs_failing_at(position: int) -> list[int]: + """Source values ``0..N-1`` with the value at ``position`` made negative.""" + return [-(i + 1) if i == position else i for i in range(N_ROWS)] + + +# --------------------------------------------------------------------------- +# Jobs +# --------------------------------------------------------------------------- + + +def source_of(xs: list[int]) -> DictSource: + """A source of rows ``k = r``, ``x = xs[i]``.""" + rows = [{"k": f"r{i}", "x": x} for i, x in enumerate(xs)] + return DictSource(data=rows, tag_columns=["k"], source_id="rows") + + +def function_job( + store: Any, xs: list[int], *, max_concurrency: int | None = None +) -> PipelineJob: + """Source -> ``f`` (fails on a negative x) -> ``down`` (doubles).""" + job = PipelineJob(name="itl879", database=store) + with job: + if max_concurrency is None: + out = checked.pod(source_of(xs), label="f") + else: + pod = FunctionPod( + PythonDataFunction(check, output_keys="y"), + pod_config=PodConfig(max_concurrency=max_concurrency), + ) + out = pod(source_of(xs), label="f") + double.pod(out, label="down") + return job + + +def ctx_function_job(store: Any, xs: list[int]) -> PipelineJob: + """Source -> ``f`` (a side-effect function pod failing on a negative x) -> ``down``.""" + job = PipelineJob(name="itl879", database=store) + with job: + out = check_with_ctx(source_of(xs), label="f") + double.pod(out, label="down") + return job + + +def side_effect_job(store: Any, xs: list[int], on_error: str) -> PipelineJob: + """Source -> side-effect node ``se`` whose delivery fails on a negative x.""" + pod = side_effect_pod( + deliver, config=SideEffectPodConfig(on_error=on_error), name="deliver" + ) + job = PipelineJob(name="itl879", database=store) + with job: + pod(source_of(xs), label="se") + return job + + +FUNCTION_JOBS: dict[str, Callable[[Any, list[int]], PipelineJob]] = { + "function": function_job, + "ctx_function": ctx_function_job, +} + + +class Recorder(NoOpObserver): + """Records ``(event, node label, row key)`` for every observer call.""" + + def __init__(self) -> None: + """Start with no events.""" + self.events: list[tuple[str, str, str | None]] = [] + + def on_node_start(self, node_label: str, node_hash: str, tag_schema: Any = None): + """Record a node start.""" + self.events.append(("node_start", node_label, None)) + + def on_node_end(self, node_label: str, node_hash: str) -> None: + """Record a node end.""" + self.events.append(("node_end", node_label, None)) + + def on_data_end(self, node_label, tag, input_data, output_data, cached) -> None: + """Record a row end.""" + self.events.append(("data_end", node_label, tag.as_dict()["k"])) + + def on_data_crash(self, node_label, tag, data, error) -> None: + """Record a row crash.""" + self.events.append(("data_crash", node_label, tag.as_dict()["k"])) + + def of(self, event: str, label: str) -> list[str | None]: + """The row keys (or ``None``) of ``event`` calls for ``label``.""" + return [k for e, lbl, k in self.events if e == event and lbl == label] + + +def run_job( + job: PipelineJob, orchestrator: str, policy: str, observer: Any = None +) -> Any: + """Run ``job`` on ``orchestrator`` with ``policy``; async runs have a timeout guard.""" + orch = ORCHESTRATORS[orchestrator](error_policy=policy) + if orchestrator == "sync": + return job.run(orchestrator=orch, observer=observer) + + async def guarded() -> Any: + async with asyncio.timeout(RUN_TIMEOUT_SECONDS): + return await job.run_async(orchestrator=orch, observer=observer) + + return asyncio.run(guarded()) + + +def ks(job: PipelineJob, label: str) -> list[str]: + """The sorted row keys node ``label`` of ``job`` reads.""" + return sorted(job.nodes[label].as_table().column("k").to_pylist()) + + +def stored(store: Any, build: Callable[[Any, list[int]], PipelineJob], label: str): + """The rows a fresh job over ``store`` reads for ``label`` without running.""" + return build(store, xs_failing_at(0)).nodes[label].as_table() + + +@pytest.fixture(params=list(ORCHESTRATORS)) +def orchestrator(request: pytest.FixtureRequest) -> str: + """Each orchestrator's name.""" + return request.param + + +@pytest.fixture(params=list(FUNCTION_JOBS)) +def build(request: pytest.FixtureRequest) -> Callable[[Any, list[int]], PipelineJob]: + """Each kind of function job.""" + return FUNCTION_JOBS[request.param] + + +@pytest.fixture +def caught_warnings() -> Iterator[list[warnings.WarningMessage]]: + """Every warning issued in the test.""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + yield caught + + +def row_failure_warnings(caught: list[warnings.WarningMessage]) -> list[str]: + """The messages of the ``RowFailuresWarning``s in ``caught``.""" + return [str(w.message) for w in caught if w.category is RowFailuresWarning] + + +# --------------------------------------------------------------------------- +# continue: already the same on both orchestrators; locked here +# --------------------------------------------------------------------------- + + +def test_continue_skips_and_reports_the_row( + orchestrator: str, build: Callable[..., PipelineJob], caught_warnings: list[Any] +) -> None: + """The failing row is skipped, counted, warned about once and seen by the observer.""" + position = FAILING_POSITIONS["middle"] + job = build(InMemoryArrowDatabase(), xs_failing_at(position)) + recorder = Recorder() + + summary = run_job(job, orchestrator, "continue", recorder) + + node = summary.nodes["f"] + assert (node.computed, node.reused, node.failed) == (N_ROWS - 1, 0, 1) + assert isinstance(node.first_error, ValueError) + assert str(node.first_error) == f"bad x={-(position + 1)}" + assert summary.nodes["down"].computed == N_ROWS - 1 + messages = row_failure_warnings(caught_warnings) + assert len(messages) == 1 + assert f"1 of {N_ROWS} rows failed in node 'f'" in messages[0] + assert recorder.of("data_crash", "f") == [f"r{position}"] + assert recorder.of("node_end", "f") == [None] + assert recorder.of("node_end", "down") == [None] + expected = [f"r{i}" for i in range(N_ROWS) if i != position] + assert ks(job, "f") == expected + assert ks(job, "down") == expected + + +# --------------------------------------------------------------------------- +# fail_fast +# --------------------------------------------------------------------------- + + +def test_fail_fast_raises_the_row_error( + orchestrator: str, build: Callable[..., PipelineJob], caught_warnings: list[Any] +) -> None: + """The run raises the row's own exception: same type and message on both orchestrators.""" + position = FAILING_POSITIONS["middle"] + job = build(InMemoryArrowDatabase(), xs_failing_at(position)) + recorder = Recorder() + + with pytest.raises(ValueError) as info: + run_job(job, orchestrator, "fail_fast", recorder) + + assert type(info.value) is ValueError + assert str(info.value) == f"bad x={-(position + 1)}" + assert getattr(info.value, "__notes__", []) == [] + assert job.last_run is None + assert row_failure_warnings(caught_warnings) == [] + assert recorder.of("data_crash", "f") == [f"r{position}"] + assert recorder.of("node_end", "f") == [None] + + +@pytest.mark.parametrize("position", list(FAILING_POSITIONS)) +def test_fail_fast_stores_only_correct_rows( + orchestrator: str, build: Callable[..., PipelineJob], position: str +) -> None: + """Every stored row is its input's correct result; the failing row is never stored.""" + store = InMemoryArrowDatabase() + xs = xs_failing_at(FAILING_POSITIONS[position]) + with pytest.raises(ValueError): + run_job(build(store, xs), orchestrator, "fail_fast") + + by_key = {f"r{i}": x for i, x in enumerate(xs)} + f_rows = stored(store, build, "f").to_pylist() + for row in f_rows: + assert row["y"] == by_key[row["k"]] >= 0 + for row in stored(store, build, "down").to_pylist(): + assert row["z"] == 2 * by_key[row["k"]] + assert by_key[row["k"]] >= 0 + if orchestrator == "sync": + # The sync run stores exactly the rows before the failing one. + assert sorted(r["k"] for r in f_rows) == [ + f"r{i}" for i in range(FAILING_POSITIONS[position]) + ] + + +@pytest.mark.parametrize("max_concurrency", [1, None]) +@pytest.mark.parametrize("position", list(FAILING_POSITIONS)) +def test_downstream_never_completes_on_partial_input( + max_concurrency: int | None, position: str +) -> None: + """The failing node's output ends early; its downstream node must not finish on it. + + The failing node closes its output as it stops. Were the downstream node to read that + as a normal end of input before the run's cancellation reached it, it would complete + on part of its input. It must be cancelled instead: no ``on_node_end``. + """ + job = function_job( + InMemoryArrowDatabase(), + xs_failing_at(FAILING_POSITIONS[position]), + max_concurrency=max_concurrency, + ) + recorder = Recorder() + + with pytest.raises(ValueError): + run_job(job, "async", "fail_fast", recorder) + + assert recorder.of("node_end", "down") == [] + + +def test_max_concurrency_one_raises_the_first_failing_row(orchestrator: str) -> None: + """With one row at a time, the first failing row in input order is the one raised.""" + xs = [0, -1, 2, -3, 4, 5] + job = function_job(InMemoryArrowDatabase(), xs, max_concurrency=1) + + with pytest.raises(ValueError, match=r"^bad x=-1$"): + run_job(job, orchestrator, "fail_fast") + + +def test_nothing_left_running_after_fail_fast( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """After the raise, no task of the run is pending and every channel is closed.""" + created: list[Any] = [] + + # The orchestrator's channels are internal; record them through its module's names. + class RecordingChannel(async_orchestrator_module.Channel): + def __post_init__(self) -> None: + super().__post_init__() + created.append(self) + + class RecordingBroadcastChannel(async_orchestrator_module.BroadcastChannel): + def __init__(self, buffer_size: int = 64) -> None: + super().__init__(buffer_size) + created.append(self) + + monkeypatch.setattr(async_orchestrator_module, "Channel", RecordingChannel) + monkeypatch.setattr( + async_orchestrator_module, "BroadcastChannel", RecordingBroadcastChannel + ) + job = PipelineJob(name="itl879", database=InMemoryArrowDatabase()) + with job: + out = checked.pod(source_of(xs_failing_at(2)), label="f") + double.pod(out, label="down") + double.pod(out, label="down2") # fan-out: a broadcast channel + + async def main() -> set[asyncio.Task[Any]]: + async with asyncio.timeout(RUN_TIMEOUT_SECONDS): + with pytest.raises(ValueError): + await job.run_async( + orchestrator=AsyncPipelineOrchestrator(error_policy="fail_fast") + ) + return asyncio.all_tasks() - {asyncio.current_task()} + + assert asyncio.run(main()) == set() + assert any(isinstance(c, RecordingBroadcastChannel) for c in created) + for channel in created: + # Closed state is private to the channel classes. + closed = channel._closed + assert closed.is_set() if isinstance(closed, asyncio.Event) else closed + + +# --------------------------------------------------------------------------- +# Side-effect pods: their own on_error, not error_policy +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("policy", ["continue", "fail_fast"]) +def test_side_effect_raise_is_the_delivery_error(orchestrator: str, policy: str) -> None: + """``on_error="raise"``: the run raises the delivery's own error, never a group.""" + job = side_effect_job(InMemoryArrowDatabase(), xs_failing_at(2), "raise") + + with pytest.raises(ValueError) as info: + run_job(job, orchestrator, policy) + + assert type(info.value) is ValueError + assert str(info.value) == "bad x=-3" + + +def test_side_effect_log_skips_and_warns( + orchestrator: str, caplog: pytest.LogCaptureFixture +) -> None: + """``on_error="log"``: logged at WARNING, the run goes on.""" + job = side_effect_job(InMemoryArrowDatabase(), xs_failing_at(2), "log") + + with caplog.at_level(logging.WARNING, logger="orcapod"): + run_job(job, orchestrator, "fail_fast") + + assert any( + r.levelno == logging.WARNING and "delivery failed: bad x=-3" in r.getMessage() + for r in caplog.records + ) + + +# --------------------------------------------------------------------------- +# Choosing the one exception of an async run +# --------------------------------------------------------------------------- + + +class _Node: + """A stand-in node: only its label is read.""" + + def __init__(self, label: str) -> None: + self.label = label + + +def test_single_failure_takes_the_earliest_node_and_notes_the_others() -> None: + """The failure of the node first in topological order wins, whatever the group order.""" + # Calls the private helper directly: the test is about its ordering rule, which an + # end-to-end run cannot force into both group orders. + from orcapod.pipeline.async_orchestrator import _single_failure + + topo_order = [_Node("up"), _Node("down")] + for order in ("up_first", "down_first"): + up_error = ValueError("up failed") + down_error = KeyError("down failed") + failed_nodes = {id(up_error): "up", id(down_error): "down"} + leaves = [up_error, down_error] if order == "up_first" else [down_error, up_error] + group = ExceptionGroup("run", [ExceptionGroup("node", leaves)]) + + chosen = _single_failure(group, topo_order, failed_nodes) + + assert chosen is up_error + assert up_error.__notes__ == [ + "Node 'down' also failed in the same run: KeyError: 'down failed'" + ] + + +def test_single_failure_alone_is_raised_unchanged() -> None: + """One failure is returned as it is, with no note.""" + # Calls the private helper directly, as above. + from orcapod.pipeline.async_orchestrator import _single_failure + + error = ValueError("only") + chosen = _single_failure( + ExceptionGroup("run", [error]), [_Node("f")], {id(error): "f"} + ) + assert chosen is error + assert getattr(error, "__notes__", None) is None From 14b818292c64a4fe73717242ff6e83bbc944d951 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:23:56 -0800 Subject: [PATCH 2/7] fix(async_orchestrator)!: honour error_policy; an async run raises the failing node's own exception (ITL-879) Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/function_pod.py | 15 ++++- src/orcapod/core/nodes/function_node.py | 26 +++++++- src/orcapod/pipeline/async_orchestrator.py | 64 +++++++++++++++++-- src/orcapod/protocols/node_protocols.py | 1 + src/orcapod/utils/store_errors.py | 20 ++++++ .../test_pipeline/test_async_error_policy.py | 16 ++++- .../test_store_errors_together.py | 2 +- 7 files changed, 134 insertions(+), 10 deletions(-) diff --git a/src/orcapod/core/function_pod.py b/src/orcapod/core/function_pod.py index a5c88259c..a2bc9e6c7 100644 --- a/src/orcapod/core/function_pod.py +++ b/src/orcapod/core/function_pod.py @@ -7,7 +7,7 @@ from abc import abstractmethod from collections.abc import Callable, Collection, Iterator, Sequence from functools import update_wrapper, wraps -from typing import TYPE_CHECKING, Any, Protocol, cast +from typing import TYPE_CHECKING, Any, Literal, Protocol, cast from orcapod import contexts from orcapod.channels import ReadableChannel, WritableChannel @@ -592,6 +592,7 @@ async def async_execute( pipeline_config: PipelineConfig | None = None, *, observer: ExecutionObserverProtocol | None = None, + error_policy: Literal["continue", "fail_fast"] = "continue", run_id: str | None = None, ) -> None: """Streaming async execution with per-data concurrency control. @@ -606,7 +607,17 @@ async def async_execute( output: Writable channel for output (tag, data) pairs. pipeline_config: Optional pipeline-level concurrency config. observer: Optional observer for per-item lifecycle hooks. + error_policy: What a row whose function raised does, as for a node's + ``execute``: ``"continue"`` skips the row; ``"fail_fast"`` raises its + exception, which cancels the rows still in flight. Rows run + concurrently, so the row raised is the first to fail, which is the + first in input order only with one row at a time. run_id: Pipeline run identifier forwarded to per-row processing. + + Raises: + Exception: Under ``"fail_fast"``, the exception of the first row whose + function raised, inside the ``ExceptionGroup`` of the pod's task group. + StoreError: A failed store read or write, whatever ``error_policy`` is. """ from orcapod.pipeline.observer import NoOpObserver @@ -647,6 +658,8 @@ async def process_one(tag: TagProtocol, data: DataProtocol) -> None: "Data processing failed, skipping: %s", exc, exc_info=True ) obs.on_data_crash(pod_label, tag, data, exc) + if error_policy == "fail_fast": + raise else: obs.on_data_end(pod_label, tag, data, result_data, cached=False) if result_data is not None: diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 186a8e384..d77643b8b 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -59,7 +59,11 @@ report_rows_of_other_data_contexts, report_unexpected_row_count, ) -from orcapod.utils.store_errors import commit_best_effort, keep_store_error +from orcapod.utils.store_errors import ( + commit_best_effort, + keep_store_error, + store_error_of, +) from orcapod.protocols.core_protocols import ( FunctionPodProtocol, DataFunctionExecutorProtocol, @@ -3300,6 +3304,7 @@ async def async_execute( output: WritableChannel[tuple[TagProtocol, DataProtocol]], *, observer: ExecutionObserverProtocol | None = None, + error_policy: Literal["continue", "fail_fast"] = "continue", run_id: str | None = None, ) -> None: """Streaming async execution for FunctionJobNode. @@ -3316,10 +3321,18 @@ async def async_execute( input_channel: Single readable channel of (tag, data) pairs. output: Writable channel for output (tag, data) pairs. observer: Optional execution observer for hooks. + error_policy: ``"continue"`` skips rows whose function raised; + ``"fail_fast"`` raises the exception of the first row whose function + raised, as ``execute`` does, and cancels the rows still in flight. + Rows run concurrently, so that is the first row to fail, not + necessarily the first in input order. run_id: Optional pipeline run identifier threaded through to ``InvocationContext.pipeline_run_id``. Raises: + Exception: Under ``"fail_fast"``, the exception of the first row whose + function raised, inside the ``ExceptionGroup`` of the node's task + group. *output* is closed first. RuntimeError: If the node was loaded by ``PipelineJob.load``: without a store it has nothing to serve, and it has no function pod to compute with. *output* is closed first. @@ -3362,11 +3375,13 @@ async def async_execute( obs = observer if observer is not None else NoOpObserver() ctx_obs = obs.contextualize(*self.node_identity_path) + node_started = False try: tag_schema = self._input_stream.output_schema( columns={"system_tags": True} )[0] ctx_obs.on_node_start(node_label, node_hash, tag_schema=tag_schema) + node_started = True if self._cached_function_pod is not None: # ---------------------------------------------------------- @@ -3583,6 +3598,7 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: [compute_channel.reader], result_channel.writer, observer=_NodeLabelObserver(), + error_policy=error_policy, run_id=run_id, ) ) @@ -3597,6 +3613,7 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: [input_channel], _CloseDeferredWriter(output), observer=ctx_obs, + error_policy=error_policy, run_id=run_id, ) @@ -3606,6 +3623,13 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: # after it has seen that close, is newer than this one (ITL-1042). self._update_modified_time() except Exception as run_error: + if node_started: + # The node ends for the observer before it raises, as in execute; a + # failing observer must not replace a store error on its way out. + with keep_store_error( + store_error_of(run_error), "Observer on_node_end" + ): + ctx_obs.on_node_end(node_label, node_hash) self._commit_after_failure(run_error) raise finally: diff --git a/src/orcapod/pipeline/async_orchestrator.py b/src/orcapod/pipeline/async_orchestrator.py index 07f53b7f7..e96bfbee5 100644 --- a/src/orcapod/pipeline/async_orchestrator.py +++ b/src/orcapod/pipeline/async_orchestrator.py @@ -191,6 +191,8 @@ async def _run_async( topo_order: list[Any] = [] store_error: StoreError | TypeIdentityError | None = None + # id of each exception a node raised -> that node's label. + failed_nodes: dict[int, str] = {} try: topo_order = list(graph.topological_sort()) buf = self._buffer_size @@ -243,6 +245,7 @@ async def _run_async( if is_source_node(node): _start_node( tg, + failed_nodes, node, node.async_execute(writer, observer=effective_observer), ) @@ -255,11 +258,13 @@ async def _run_async( input_reader = edge_readers[(predecessors[0], node)] _start_node( tg, + failed_nodes, node, node.async_execute( input_reader, writer, observer=effective_observer, + error_policy=self._error_policy, run_id=run_id, ), ) @@ -278,6 +283,7 @@ async def _run_async( ] _start_node( tg, + failed_nodes, node, node.async_execute( input_readers, writer, observer=effective_observer @@ -292,6 +298,7 @@ async def _run_async( input_reader = edge_readers[(predecessors[0], node)] _start_node( tg, + failed_nodes, node, node.async_execute( [input_reader], @@ -327,7 +334,9 @@ async def _run_async( if identity_errors: store_error = identity_errors[0] raise store_error - raise + # Any other failure is raised on its own too, as the sync orchestrator + # raises it: a row's error under fail_fast, a side effect's delivery error. + raise _single_failure(group, topo_order, failed_nodes) finally: # A failing on_run_end must not replace a failed store read or write. with keep_store_error(store_error, "Observer on_run_end"): @@ -596,15 +605,22 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: def _start_node( - tg: asyncio.TaskGroup, node: Any, work: Coroutine[Any, Any, None] + tg: asyncio.TaskGroup, + failed_nodes: dict[int, str], + node: Any, + work: Coroutine[Any, Any, None], ) -> None: - """Start a node's coroutine in ``tg``, naming the node on the store errors it raises. + """Start a node's coroutine in ``tg``, recording which node each failure came from. A store error raised on its own or inside an exception group (a node that runs - its own task group) gets the node's label when it has none. + its own task group) gets the node's label when it has none. Every exception the + node raises, each one inside a group included, is recorded in ``failed_nodes`` + under the node's label, so ``_single_failure`` can order the run's failures as + the sync orchestrator meets them. Args: tg: The run's task group. + failed_nodes: The run's record, by exception id, of the node that raised it. node: The node the coroutine runs. work: The node's ``async_execute`` coroutine. """ @@ -616,17 +632,57 @@ async def run() -> None: # Name the node the failure belongs to, if the raising site did not, # so _single_store_error can order it like the sync orchestrator. exc.add_context(node_label=node.label) + failed_nodes[id(exc)] = node.label raise except BaseExceptionGroup as group: # A node that runs its own task group raises its store errors inside # a group; name the node on each and re-raise the group unchanged. for store_error in StoreError.in_group(group): store_error.add_context(node_label=node.label) + for leaf in _leaves(group): + failed_nodes[id(leaf)] = node.label + raise + except Exception as exc: + failed_nodes[id(exc)] = node.label raise tg.create_task(run()) +def _single_failure( + group: BaseExceptionGroup, topo_order: list[Any], failed_nodes: dict[int, str] +) -> BaseException: + """Pick the one exception an async run raises when no store error is among its failures. + + The caller gets one exception, as from the sync orchestrator, never the task + group's exception group: the one whose node comes first in topological order (the + one the sync orchestrator would meet first, when both failures happen), with ties + and exceptions of no node broken by their order in the group, which is the order + the failures happened in. Every other exception is noted on the chosen one; the + whole group becomes its ``__context__`` when it is raised. + + Args: + group: The exception group the run's ``TaskGroup`` raised. + topo_order: The run's nodes in topological order. + failed_nodes: The node label of each exception a node raised, by exception id. + + Returns: + The exception to raise. + """ + leaves = _leaves(group) + rank = {getattr(node, "label", None): i for i, node in enumerate(topo_order)} + chosen = min(leaves, key=lambda exc: rank.get(failed_nodes.get(id(exc)), len(rank))) + for other in leaves: + if other is chosen: + continue + label = failed_nodes.get(id(other)) + where = f"Node {label!r}" if label is not None else "Another task" + chosen.add_note( + f"{where} also failed in the same run: {type(other).__name__}: {other}" + ) + return chosen + + def _single_store_error( group: ExceptionGroup, topo_order: list[Any] ) -> StoreError | None: diff --git a/src/orcapod/protocols/node_protocols.py b/src/orcapod/protocols/node_protocols.py index 98894e2e8..4807a382b 100644 --- a/src/orcapod/protocols/node_protocols.py +++ b/src/orcapod/protocols/node_protocols.py @@ -96,6 +96,7 @@ async def async_execute( output: WritableChannel[tuple[TagProtocol, DataProtocol]], *, observer: ExecutionObserverProtocol | None = None, + error_policy: Literal["continue", "fail_fast"] = "continue", run_id: str | None = None, ) -> None: ... diff --git a/src/orcapod/utils/store_errors.py b/src/orcapod/utils/store_errors.py index 5ba8ad4d6..d297f024d 100644 --- a/src/orcapod/utils/store_errors.py +++ b/src/orcapod/utils/store_errors.py @@ -45,6 +45,26 @@ def note_callback_failure( ) +def store_error_of(error: BaseException) -> StoreError | TypeIdentityError | None: + """Return the store error a failure carries: itself, or the first inside its group. + + Args: + error: A run's failure, possibly an exception group from a task group. + + Returns: + The ``StoreError`` or ``TypeIdentityError`` that is ``error`` or is the first + one found inside it, or ``None`` when it carries none. + """ + if isinstance(error, (StoreError, TypeIdentityError)): + return error + if isinstance(error, BaseExceptionGroup): + for inner in error.exceptions: + found = store_error_of(inner) + if found is not None: + return found + return None + + @contextmanager def keep_store_error( store_error: StoreError | TypeIdentityError | None, during: str diff --git a/tests/test_pipeline/test_async_error_policy.py b/tests/test_pipeline/test_async_error_policy.py index a4ce675ed..92924b4c8 100644 --- a/tests/test_pipeline/test_async_error_policy.py +++ b/tests/test_pipeline/test_async_error_policy.py @@ -65,6 +65,12 @@ def double(y: int) -> int: return 2 * y +@function_pod(output_keys="w") +def triple(y: int) -> int: + """Return three times ``y``.""" + return 3 * y + + @side_effect_function_pod(output_keys="y") def check_with_ctx(x: int, ctx: InvocationContext) -> int: """Return ``x``; raise for a negative ``x`` (a pod that takes the invocation context).""" @@ -355,7 +361,7 @@ def __init__(self, buffer_size: int = 64) -> None: with job: out = checked.pod(source_of(xs_failing_at(2)), label="f") double.pod(out, label="down") - double.pod(out, label="down2") # fan-out: a broadcast channel + triple.pod(out, label="down2") # fan-out: a broadcast channel async def main() -> set[asyncio.Task[Any]]: async with asyncio.timeout(RUN_TIMEOUT_SECONDS): @@ -379,7 +385,9 @@ async def main() -> set[asyncio.Task[Any]]: @pytest.mark.parametrize("policy", ["continue", "fail_fast"]) -def test_side_effect_raise_is_the_delivery_error(orchestrator: str, policy: str) -> None: +def test_side_effect_raise_is_the_delivery_error( + orchestrator: str, policy: str +) -> None: """``on_error="raise"``: the run raises the delivery's own error, never a group.""" job = side_effect_job(InMemoryArrowDatabase(), xs_failing_at(2), "raise") @@ -428,7 +436,9 @@ def test_single_failure_takes_the_earliest_node_and_notes_the_others() -> None: up_error = ValueError("up failed") down_error = KeyError("down failed") failed_nodes = {id(up_error): "up", id(down_error): "down"} - leaves = [up_error, down_error] if order == "up_first" else [down_error, up_error] + leaves = ( + [up_error, down_error] if order == "up_first" else [down_error, up_error] + ) group = ExceptionGroup("run", [ExceptionGroup("node", leaves)]) chosen = _single_failure(group, topo_order, failed_nodes) diff --git a/tests/test_pipeline/test_store_errors_together.py b/tests/test_pipeline/test_store_errors_together.py index b70c5f1a0..1e3162b26 100644 --- a/tests/test_pipeline/test_store_errors_together.py +++ b/tests/test_pipeline/test_store_errors_together.py @@ -354,7 +354,7 @@ async def node_work() -> None: async def main() -> None: async with asyncio.TaskGroup() as tg: - _start_node(tg, Node(), node_work()) + _start_node(tg, {}, Node(), node_work()) with pytest.raises(ExceptionGroup): asyncio.run(main()) From d6524c9625bf83fbb16c78b00d9135ee2acd2e3e Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 21:27:22 -0800 Subject: [PATCH 3/7] docs(function_pods): stopping at the first failed row on both orchestrators; changelog and design issues (ITL-879) Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-879.md | 9 +++ design_issues/ITL-878-SW1.md | 5 +- design_issues/ITL-879-EP1.md | 35 +++++++++++ docs/concepts/function-pods.md | 62 ++++++++++++++++--- src/orcapod/pipeline/async_orchestrator.py | 2 +- .../test_pipeline/test_async_error_policy.py | 34 +++++----- 6 files changed, 123 insertions(+), 24 deletions(-) create mode 100644 changelog.d/ITL-879.md create mode 100644 design_issues/ITL-879-EP1.md diff --git a/changelog.d/ITL-879.md b/changelog.d/ITL-879.md new file mode 100644 index 000000000..7d65b0492 --- /dev/null +++ b/changelog.d/ITL-879.md @@ -0,0 +1,9 @@ +### Breaking changes and migration + +- `AsyncPipelineOrchestrator(error_policy="fail_fast")` now stops the run at the first row whose function raises and raises that row's own exception, as `SyncPipelineOrchestrator` does (ITL-879). Before, the async orchestrator ignored the policy: the run skipped the row, computed the others and returned. An async run with `fail_fast` that used to finish now raises; pass `error_policy="continue"` (the default) to keep the old behaviour. Rows run concurrently, so the row raised is the first to fail, which is the first in input order only with `max_concurrency=1`. Every stored row is a correct result of its input row and the failing row is never stored; which other rows are stored before the run stops is not specified (see `docs/concepts/function-pods.md`, "Stopping at the first failed row"). +- An async run raises the failing node's own exception, never the task group's `ExceptionGroup` (ITL-879). This covers a side-effect pod with `on_error="raise"`, an operator that rejects its inputs, and any other node failure, as store errors already were. When several nodes fail before the run stops, the exception of the node first in the pipeline is raised and the others are added to it as notes (`error.__notes__`). Replace `except* SomeError` or `except ExceptionGroup` around `job.run()` / `job.run_async()` / `AsyncPipelineOrchestrator.run()` with `except SomeError`. +- `FunctionNodeProtocol.async_execute`, `FunctionJobNode.async_execute` and the function pods' `async_execute` take a keyword-only `error_policy` (default `"continue"`), as `execute` already did. A class implementing `FunctionNodeProtocol` must accept it. + +### Fixed + +- Under the async orchestrator a function node whose run stops on a row failure now calls the observer's `on_node_end` before raising, as the sync orchestrator does (ITL-879). diff --git a/design_issues/ITL-878-SW1.md b/design_issues/ITL-878-SW1.md index 8d178b5a6..385cf8bb8 100644 --- a/design_issues/ITL-878-SW1.md +++ b/design_issues/ITL-878-SW1.md @@ -39,7 +39,8 @@ cannot be stored (a naive `datetime`, a value not matching the declared type, re `ResultCache.store` prepares the record, before any write) is a row failure governed by `error_policy` exactly like a function that raised, not a `StoreWriteError`: the value is specific to that row, the store is healthy, and every re-run would fail at the same row. The -async orchestrator logs row failures at DEBUG and ignores `fail_fast` (ITL-879), unchanged here. +async orchestrator ignored `fail_fast` for such rows (ITL-879), unchanged here and fixed since +by ITL-879 (`ITL-879-EP1`). Regressions: `tests/test_pipeline/test_store_write_failure.py`. Related, not fixed here: the async -orchestrator ignores `error_policy` (ITL-879); connector and Delta `flush` drop a batch whose +orchestrator ignored `error_policy` (ITL-879, since fixed: `ITL-879-EP1`); connector and Delta `flush` drop a batch whose write fails (ITL-880). diff --git a/design_issues/ITL-879-EP1.md b/design_issues/ITL-879-EP1.md new file mode 100644 index 000000000..f981f32b4 --- /dev/null +++ b/design_issues/ITL-879-EP1.md @@ -0,0 +1,35 @@ +## `src/orcapod/pipeline/async_orchestrator.py` — error policy on the async path + +### EP1 — The async orchestrator ignored `error_policy`; a node failure reached the caller as an `ExceptionGroup` +**Status:** resolved +**Severity:** high +**Issue:** ITL-879 + +`AsyncPipelineOrchestrator` stored `error_policy` and never used it. The pod's per-row +handler (`_FunctionPodBase.async_execute.process_one`) skipped every row whose function +raised, so under `error_policy="fail_fast"` an async run computed every other row and +finished, with a `RunSummary` and a `RowFailuresWarning`, where the sync run raised the row's +exception after storing the rows before it. Any other node failure that was not a store +error (a side-effect pod with `on_error="raise"`, an operator that rejected its inputs) +reached the caller as the task group's `ExceptionGroup`, where the sync run raised the +exception itself. Under `"continue"` the two orchestrators already agreed: no raise, the +same summary, one `RowFailuresWarning`, the per-row log at DEBUG on both (the issue's +"WARNING" premise predates ITL-997), the same observer events and stored rows. + +**Fix:** `error_policy` is a keyword-only argument (default `"continue"`) of +`FunctionNodeProtocol.async_execute`, `FunctionJobNode.async_execute` and +`_FunctionPodBase.async_execute`, as of `execute` (coordinator decision, November 13, 2026), +and the orchestrator passes its own. Under `"fail_fast"` the pod re-raises the row's +exception after `on_data_crash`; the existing task groups cancel the rows and nodes still +running and every node closes its output in its existing `finally`; the failing node calls +`on_node_end` before it raises, as `execute` does. The orchestrator's cancel and close +machinery is unchanged. Its exception-group handler now always raises ONE exception: a store +error, then a `TypeIdentityError` (both as before), then the exception of the node earliest +in topological order (ties: the order the failures happened), the others noted on it +(`_single_failure`). Rows run concurrently, so the row raised is the first to fail, the first +in input order only with `max_concurrency=1`. Storage guarantee, as documented in +`docs/concepts/function-pods.md`: every stored row is a correct, complete result of its input +row; the failing row is never stored; which other rows are stored is not specified. No +hash, golden value or stored layout changed. Regressions: +`tests/test_pipeline/test_async_error_policy.py`. Related, not changed: a pod-level error +policy (F15, ITL-527). diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 108453b24..d34165463 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -215,9 +215,8 @@ Fix the function (`return round(weight / 2)`), or declare the type it really ret `OutputTypeError` is a `TypeError` (and a `ValueError`). Inside a `PipelineJob` it counts as a failure of the function for that row, like an exception the function raises itself. With the default `error_policy="continue"` the row is skipped and reported to the observer, and -the other rows are computed and stored. With `error_policy="fail_fast"` the synchronous -orchestrator stops the run; the asynchronous orchestrator does not stop for row failures yet -(ITL-879, see below). +the other rows are computed and stored. With `error_policy="fail_fast"` the run stops and +raises it, on either orchestrator (see "Stopping at the first failed row" below). The function-output rule is: the returned value must be an instance of the declared type, or an exact numeric equivalent; a subclass stands in for its base. What is stored for a subclass instance is decided by the logical type of the declared class: a user class is stored as the base (the declared class's fields only), while a `torch.nn.Parameter` returned for `-> torch.Tensor` is stored, read back and hashed as an `nn.Parameter`. @@ -514,8 +513,8 @@ print(summary) `job.last_run.print_traceback("bmi")` prints the full traceback of the node's first failed row, ending in the line of your function that raised. Fix the function and run the job again: the rows that succeeded before are read from the store, and only the failed ones -are computed. To stop a run at the first failed row instead, pass -`SyncPipelineOrchestrator(error_policy="fail_fast")` as the orchestrator. +are computed. To stop a run at the first failed row instead, see "Stopping at the first +failed row" below. The counts are also available one by one, for example in a test: @@ -527,6 +526,56 @@ print(summary.ok, summary.failed_rows) # False 3 ``` +### Stopping at the first failed row + +Pass an orchestrator with `error_policy="fail_fast"` to stop the run at the first row whose +function raises. The run raises that row's own exception, on either orchestrator: the same +type and message, never an exception group. Nothing is skipped silently, no +`RowFailuresWarning` is issued and no summary is returned (`job.last_run` keeps the previous +run's). + +```python +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator + +for orchestrator in (SyncPipelineOrchestrator, AsyncPipelineOrchestrator): + job = PipelineJob(name="bmi_typo_fast", database=InMemoryArrowDatabase()) + with job: + source = DictSource(data=mice, tag_columns=["subject_id"], source_id="mice") + careless_bmi.pod(source, label="bmi") + try: + job.run(orchestrator=orchestrator(error_policy="fail_fast")) + except NameError as error: + print(type(error).__name__, error) +# NameError name 'heigth' is not defined +# NameError name 'heigth' is not defined +``` + +The synchronous orchestrator computes one row at a time, so it raises the first failing row +in input order, and the rows before it are computed and stored; the nodes after it never +start. The asynchronous orchestrator computes rows concurrently (up to the pod's +`max_concurrency`) and nodes run side by side: it raises the first row *to fail*, which is the +first in input order only with `max_concurrency=1`, cancels every row and node still running, +and closes their channels before it raises. A function already running in a worker thread +cannot be interrupted: its result is discarded, never stored, and `job.run()` returns once it +has finished. When rows of several nodes fail before the run stops, the error raised is the +one of the node that comes first in the pipeline; the others are noted on it +(`error.__notes__`). + +What is stored after such a run: + +After a fail_fast raise, every stored row is a correct, complete result of its own input +row, written in the ITL-878 commit order (results before pipeline records; writes that +succeeded are flushed before the raise). The failing row is never stored. Other rows of the +failing node, before or after it in input order, rows that downstream nodes computed from its +successful rows, and rows of other branches may be stored; the set stored is not specified +beyond that. When the cancellation falls between a row's result write and its pipeline-record +write, the result is stored without its record: a re-run reuses it without calling the +function, as after a crash. + +A side-effect pod does not follow `error_policy`: its own `on_error` decides. With +`on_error="raise"` (the default) the run raises the delivery's own exception, on either +orchestrator; with `on_error="log"` the failure is logged at WARNING and the run goes on. + ### When results are committed Inside `job.run()`, each function node commits its results and pipeline records to the @@ -643,8 +692,7 @@ prepared, before anything is written, and it is a *row failure*: it is handled e function that raised on that row. The observer gets `on_data_crash`, the failure is logged, and `error_policy` decides -- `"continue"` skips the row, `"fail_fast"` raises the error the conversion raised. It is not a `StoreWriteError`, because the store is healthy and every -re-run would fail at the same row. The asynchronous orchestrator currently logs row failures -at DEBUG and does not stop for `"fail_fast"` (ITL-879); this applies to these rows too. +re-run would fail at the same row. Both orchestrators handle these rows the same way. This is the contract for now. An opt-in mode that tolerates store failures may be added later. diff --git a/src/orcapod/pipeline/async_orchestrator.py b/src/orcapod/pipeline/async_orchestrator.py index e96bfbee5..e9c28147e 100644 --- a/src/orcapod/pipeline/async_orchestrator.py +++ b/src/orcapod/pipeline/async_orchestrator.py @@ -65,7 +65,7 @@ def __init__( error_policy: Literal["continue", "fail_fast"] = "continue", ) -> None: self._buffer_size = buffer_size - self._error_policy = error_policy + self._error_policy: Literal["continue", "fail_fast"] = error_policy def run( self, diff --git a/tests/test_pipeline/test_async_error_policy.py b/tests/test_pipeline/test_async_error_policy.py index 92924b4c8..0fc2b24bd 100644 --- a/tests/test_pipeline/test_async_error_policy.py +++ b/tests/test_pipeline/test_async_error_policy.py @@ -16,19 +16,18 @@ import logging import warnings from collections.abc import Callable, Iterator -from typing import Any +from typing import Any, Literal import pytest from orcapod import PipelineJob, function_pod -from orcapod.core.data_function import PythonDataFunction -from orcapod.core.function_pod import FunctionPod, side_effect_function_pod +from orcapod.core.function_pod import FunctionPod from orcapod.databases import InMemoryArrowDatabase from orcapod.errors import RowFailuresWarning from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator from orcapod.pipeline import async_orchestrator as async_orchestrator_module from orcapod.pipeline.observer import NoOpObserver -from orcapod.side_effects import InvocationContext, SideEffectPodConfig, side_effect_pod +from orcapod.side_effects import InvocationContext, SideEffectPod, SideEffectPodConfig from orcapod.sources import DictSource from orcapod.types import PodConfig @@ -71,7 +70,6 @@ def triple(y: int) -> int: return 3 * y -@side_effect_function_pod(output_keys="y") def check_with_ctx(x: int, ctx: InvocationContext) -> int: """Return ``x``; raise for a negative ``x`` (a pod that takes the invocation context).""" if x < 0: @@ -79,6 +77,11 @@ def check_with_ctx(x: int, ctx: InvocationContext) -> int: return x +checked_with_ctx = FunctionPod.from_fn( + check_with_ctx, output_keys="y", ctx_arg_name="ctx" +) + + def deliver(x: int, ctx: InvocationContext) -> None: """A side effect that fails for a negative ``x``.""" if x < 0: @@ -110,8 +113,9 @@ def function_job( if max_concurrency is None: out = checked.pod(source_of(xs), label="f") else: - pod = FunctionPod( - PythonDataFunction(check, output_keys="y"), + pod = FunctionPod.from_fn( + check, + output_keys="y", pod_config=PodConfig(max_concurrency=max_concurrency), ) out = pod(source_of(xs), label="f") @@ -123,14 +127,16 @@ def ctx_function_job(store: Any, xs: list[int]) -> PipelineJob: """Source -> ``f`` (a side-effect function pod failing on a negative x) -> ``down``.""" job = PipelineJob(name="itl879", database=store) with job: - out = check_with_ctx(source_of(xs), label="f") + out = checked_with_ctx(source_of(xs), label="f") double.pod(out, label="down") return job -def side_effect_job(store: Any, xs: list[int], on_error: str) -> PipelineJob: +def side_effect_job( + store: Any, xs: list[int], on_error: Literal["raise", "log"] +) -> PipelineJob: """Source -> side-effect node ``se`` whose delivery fails on a negative x.""" - pod = side_effect_pod( + pod = SideEffectPod( deliver, config=SideEffectPodConfig(on_error=on_error), name="deliver" ) job = PipelineJob(name="itl879", database=store) @@ -162,11 +168,11 @@ def on_node_end(self, node_label: str, node_hash: str) -> None: def on_data_end(self, node_label, tag, input_data, output_data, cached) -> None: """Record a row end.""" - self.events.append(("data_end", node_label, tag.as_dict()["k"])) + self.events.append(("data_end", node_label, str(tag.as_dict()["k"]))) def on_data_crash(self, node_label, tag, data, error) -> None: """Record a row crash.""" - self.events.append(("data_crash", node_label, tag.as_dict()["k"])) + self.events.append(("data_crash", node_label, str(tag.as_dict()["k"]))) def of(self, event: str, label: str) -> list[str | None]: """The row keys (or ``None``) of ``event`` calls for ``label``.""" @@ -343,12 +349,12 @@ def test_nothing_left_running_after_fail_fast( created: list[Any] = [] # The orchestrator's channels are internal; record them through its module's names. - class RecordingChannel(async_orchestrator_module.Channel): + class RecordingChannel(async_orchestrator_module.Channel[Any]): def __post_init__(self) -> None: super().__post_init__() created.append(self) - class RecordingBroadcastChannel(async_orchestrator_module.BroadcastChannel): + class RecordingBroadcastChannel(async_orchestrator_module.BroadcastChannel[Any]): def __init__(self, buffer_size: int = 64) -> None: super().__init__(buffer_size) created.append(self) From de60388da0cec14e5fb55e66d651699e18329cce Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 22:34:09 -0800 Subject: [PATCH 4/7] fix(function_node): a node cancelled by another node's failure commits what it wrote; state the error priority (ITL-879) Addresses CodeRabbit round 1 on PR #379. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-879.md | 3 +- design_issues/ITL-879-EP1.md | 5 +- docs/concepts/function-pods.md | 8 +- src/orcapod/core/nodes/function_node.py | 6 ++ .../test_pipeline/test_job_batched_commits.py | 92 +++++++++++++++++-- 5 files changed, 102 insertions(+), 12 deletions(-) diff --git a/changelog.d/ITL-879.md b/changelog.d/ITL-879.md index 7d65b0492..64a1a1abf 100644 --- a/changelog.d/ITL-879.md +++ b/changelog.d/ITL-879.md @@ -1,9 +1,10 @@ ### Breaking changes and migration - `AsyncPipelineOrchestrator(error_policy="fail_fast")` now stops the run at the first row whose function raises and raises that row's own exception, as `SyncPipelineOrchestrator` does (ITL-879). Before, the async orchestrator ignored the policy: the run skipped the row, computed the others and returned. An async run with `fail_fast` that used to finish now raises; pass `error_policy="continue"` (the default) to keep the old behaviour. Rows run concurrently, so the row raised is the first to fail, which is the first in input order only with `max_concurrency=1`. Every stored row is a correct result of its input row and the failing row is never stored; which other rows are stored before the run stops is not specified (see `docs/concepts/function-pods.md`, "Stopping at the first failed row"). -- An async run raises the failing node's own exception, never the task group's `ExceptionGroup` (ITL-879). This covers a side-effect pod with `on_error="raise"`, an operator that rejects its inputs, and any other node failure, as store errors already were. When several nodes fail before the run stops, the exception of the node first in the pipeline is raised and the others are added to it as notes (`error.__notes__`). Replace `except* SomeError` or `except ExceptionGroup` around `job.run()` / `job.run_async()` / `AsyncPipelineOrchestrator.run()` with `except SomeError`. +- An async run raises the failing node's own exception, never the task group's `ExceptionGroup` (ITL-879). This covers a side-effect pod with `on_error="raise"`, an operator that rejects its inputs, and any other node failure, as store errors already were. When several nodes fail before the run stops, one exception is raised and the others are added to it as notes (`error.__notes__`): a store error (`StoreError`) first, then a stored row that cannot be decoded as its class (`TypeIdentityError`), as before; among the other failures, the one of the node that comes first in the pipeline. Replace `except* SomeError` or `except ExceptionGroup` around `job.run()` / `job.run_async()` / `AsyncPipelineOrchestrator.run()` with `except SomeError`. - `FunctionNodeProtocol.async_execute`, `FunctionJobNode.async_execute` and the function pods' `async_execute` take a keyword-only `error_policy` (default `"continue"`), as `execute` already did. A class implementing `FunctionNodeProtocol` must accept it. ### Fixed - Under the async orchestrator a function node whose run stops on a row failure now calls the observer's `on_node_end` before raising, as the sync orchestrator does (ITL-879). +- A function node that the async orchestrator cancels because another node failed now commits the results it wrote before the cancellation propagates, as a failing node does (ITL-879). Before, its ephemeral results were never committed while the job committed its pipeline records, and on a Delta store its results were not committed ahead of its pipeline records. diff --git a/design_issues/ITL-879-EP1.md b/design_issues/ITL-879-EP1.md index f981f32b4..dca7d2ede 100644 --- a/design_issues/ITL-879-EP1.md +++ b/design_issues/ITL-879-EP1.md @@ -26,7 +26,10 @@ running and every node closes its output in its existing `finally`; the failing machinery is unchanged. Its exception-group handler now always raises ONE exception: a store error, then a `TypeIdentityError` (both as before), then the exception of the node earliest in topological order (ties: the order the failures happened), the others noted on it -(`_single_failure`). Rows run concurrently, so the row raised is the first to fail, the first +(`_single_failure`). A function node cancelled by another node's failure commits what it +wrote (`_commit_after_failure`) before the cancellation propagates, as a failing node does: +the job's flush commits neither its ephemeral store nor its result table ahead of its +pipeline records (CodeRabbit, PR #379). Rows run concurrently, so the row raised is the first to fail, the first in input order only with `max_concurrency=1`. Storage guarantee, as documented in `docs/concepts/function-pods.md`: every stored row is a correct, complete result of its input row; the failing row is never stored; which other rows are stored is not specified. No diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index d34165463..cacd201e1 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -557,9 +557,11 @@ start. The asynchronous orchestrator computes rows concurrently (up to the pod's first in input order only with `max_concurrency=1`, cancels every row and node still running, and closes their channels before it raises. A function already running in a worker thread cannot be interrupted: its result is discarded, never stored, and `job.run()` returns once it -has finished. When rows of several nodes fail before the run stops, the error raised is the -one of the node that comes first in the pipeline; the others are noted on it -(`error.__notes__`). +has finished. When rows of several nodes fail before the run stops, one error is raised and +the others are noted on it (`error.__notes__`): a store error first (see "When the store +cannot be written" below), then a `TypeIdentityError`, and otherwise the error of the node +that comes first in the pipeline. A node cancelled by the failure commits the results it +wrote before it stops. What is stored after such a run: diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index d77643b8b..8af74ac0e 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -3622,6 +3622,12 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: # the finally closes the output, so every downstream node, which stamps # after it has seen that close, is newer than this one (ITL-1042). self._update_modified_time() + except asyncio.CancelledError as cancelled: + # Another node's failure cancelled this one: commit what it wrote, as a + # failing node does. The job's flush commits neither a node's ephemeral + # store nor its result table ahead of its pipeline records. + self._commit_after_failure(cancelled) + raise except Exception as run_error: if node_started: # The node ends for the observer before it raises, as in execute; a diff --git a/tests/test_pipeline/test_job_batched_commits.py b/tests/test_pipeline/test_job_batched_commits.py index 5df22828d..874e054f5 100644 --- a/tests/test_pipeline/test_job_batched_commits.py +++ b/tests/test_pipeline/test_job_batched_commits.py @@ -25,6 +25,7 @@ from orcapod import PipelineJob, function_pod from orcapod.core.cached_function_pod import CachedFunctionPod +from orcapod.core.function_pod import FunctionPod from orcapod.core.nodes.function_node import FunctionJobNode from orcapod.contexts import get_default_context from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase @@ -35,7 +36,7 @@ from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator from orcapod.databases.utils import as_table_committing from orcapod.sources import DictSource -from orcapod.types import NodeConfig +from orcapod.types import NodeConfig, PodConfig ORCHESTRATORS: dict[str, Callable[..., Any]] = { "sync": SyncPipelineOrchestrator, @@ -239,21 +240,98 @@ def ephemeral_rows(path: Path) -> int: return stored_rows(table) +@pytest.mark.parametrize("orchestrator", list(ORCHESTRATORS)) def test_a_node_failing_on_a_row_commits_its_ephemeral_results( - tmp_path: Path, + tmp_path: Path, orchestrator: str ) -> None: - """A function error under ``fail_fast``: rows before it are in the ephemeral store.""" + """A function error under ``fail_fast``: rows computed before it are in the ephemeral store.""" job = build_job(DeltaTableDatabase(tmp_path / "store"), 10, pod=double_until_7) job.set_ephemeral_store(DeltaTableDatabase(tmp_path / "ephemeral")) job.apply_node_config(NodeConfig(is_result_ephemeral=True)) - # The sync orchestrator only: the async one does not apply error_policy and - # skips a row whose function raised. with pytest.raises(ValueError, match="x is 7"): job.run( - orchestrator=SyncPipelineOrchestrator(error_policy="fail_fast"), + orchestrator=ORCHESTRATORS[orchestrator](error_policy="fail_fast"), + commit_every_rows=100, + ) + committed = ephemeral_rows(tmp_path / "ephemeral") + if orchestrator == "sync": + assert committed == 7 # rows 0 to 6 + else: + # Rows run concurrently: any of the other 9 rows may have been computed. + assert committed <= 9 + + +@function_pod(output_keys="y") +def double_until_last(x: int) -> int: + """Return twice ``x``, failing for the last row of ``CANCEL_ROWS``.""" + if x == CANCEL_ROWS - 1: + raise ValueError("last row") + return 2 * x + + +@function_pod(output_keys="z") +def plus_one(y: int) -> int: + """Return ``y + 1``.""" + return y + 1 + + +CANCEL_ROWS = 30 + + +def delta_rows_named(root: Path, name: str) -> int: + """Rows committed to the Delta table under ``root`` whose path names ``name``. + + Zero when no such table was committed. + """ + tables = { + log.parent + for log in root.rglob("_delta_log") + if name in log.parent.relative_to(root).parts + } + if not tables: + return 0 + (table,) = tables + return stored_rows(table) + + +def test_a_node_cancelled_by_a_failure_commits_its_ephemeral_results( + tmp_path: Path, +) -> None: + """A node cancelled because another node failed commits what it wrote first. + + ``inc`` runs on the rows ``dbl`` sends it until ``dbl`` fails at its last row under + ``fail_fast``; the run then cancels ``inc``. The job commits the store's pipeline + records afterwards but not a node's ephemeral store, so the cancelled node must + commit its ephemeral results itself: every pipeline record it left must have its + result. + """ + job = PipelineJob(name="batched", database=DeltaTableDatabase(tmp_path / "store")) + with job: + source = DictSource( + data=[{"k": i, "x": i} for i in range(CANCEL_ROWS)], + tag_columns=["k"], + source_id="src", + ) + upstream = FunctionPod.from_fn( + double_until_last, + output_keys="y", + pod_config=PodConfig(max_concurrency=1), + ) + plus_one.pod(upstream(source, label="dbl"), label="inc") + job.set_ephemeral_store(DeltaTableDatabase(tmp_path / "ephemeral")) + # Only the cancelled node keeps its results in the ephemeral store, so the failing + # node's own commit after its failure does not flush that store for it. + job.nodes["inc"].node_config = NodeConfig(is_result_ephemeral=True) + + with pytest.raises(ValueError, match="last row"): + job.run( + orchestrator=AsyncPipelineOrchestrator(error_policy="fail_fast"), commit_every_rows=100, ) - assert ephemeral_rows(tmp_path / "ephemeral") == 7 # rows 0 to 6 + + records = delta_rows_named(tmp_path / "store", "plus_one") + assert records > 0, "the downstream node recorded nothing; the test proves nothing" + assert delta_rows_named(tmp_path / "ephemeral", "plus_one") >= records class FailPipelineRecords: From 96837aaa6e066acdb6e99727e46b2689f98aaaf8 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 23:17:53 -0800 Subject: [PATCH 5/7] fix(function_node): an observer failing in on_node_end does not skip a failed node's commit (ITL-879) Addresses the retained concern of CodeRabbit round 2 on PR #379. Co-Authored-By: Claude Opus 5.5 --- design_issues/ITL-879-EP1.md | 3 +- src/orcapod/core/nodes/function_node.py | 19 ++++---- .../test_pipeline/test_job_batched_commits.py | 47 +++++++++++++++++++ 3 files changed, 60 insertions(+), 9 deletions(-) diff --git a/design_issues/ITL-879-EP1.md b/design_issues/ITL-879-EP1.md index dca7d2ede..c450fda40 100644 --- a/design_issues/ITL-879-EP1.md +++ b/design_issues/ITL-879-EP1.md @@ -29,7 +29,8 @@ in topological order (ties: the order the failures happened), the others noted o (`_single_failure`). A function node cancelled by another node's failure commits what it wrote (`_commit_after_failure`) before the cancellation propagates, as a failing node does: the job's flush commits neither its ephemeral store nor its result table ahead of its -pipeline records (CodeRabbit, PR #379). Rows run concurrently, so the row raised is the first to fail, the first +pipeline records (CodeRabbit, PR #379). An observer whose `on_node_end` raises as the node +fails does not skip that commit either, as on the sync path. Rows run concurrently, so the row raised is the first to fail, the first in input order only with `max_concurrency=1`. Storage guarantee, as documented in `docs/concepts/function-pods.md`: every stored row is a correct, complete result of its input row; the failing row is never stored; which other rows are stored is not specified. No diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 8af74ac0e..30d343bac 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -3629,14 +3629,17 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: self._commit_after_failure(cancelled) raise except Exception as run_error: - if node_started: - # The node ends for the observer before it raises, as in execute; a - # failing observer must not replace a store error on its way out. - with keep_store_error( - store_error_of(run_error), "Observer on_node_end" - ): - ctx_obs.on_node_end(node_label, node_hash) - self._commit_after_failure(run_error) + try: + if node_started: + # The node ends for the observer before it raises, as in execute; + # a failing observer must not replace a store error on its way out. + with keep_store_error( + store_error_of(run_error), "Observer on_node_end" + ): + ctx_obs.on_node_end(node_label, node_hash) + finally: + # Also when the observer raised instead, as execute commits then. + self._commit_after_failure(run_error) raise finally: await output.close() diff --git a/tests/test_pipeline/test_job_batched_commits.py b/tests/test_pipeline/test_job_batched_commits.py index 874e054f5..2bfbd6f44 100644 --- a/tests/test_pipeline/test_job_batched_commits.py +++ b/tests/test_pipeline/test_job_batched_commits.py @@ -33,6 +33,7 @@ from orcapod.databases.storage_utils import delta_to_pyarrow_dataset from orcapod.errors import StoreWriteError from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator +from orcapod.pipeline.observer import NoOpObserver from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator from orcapod.databases.utils import as_table_committing from orcapod.sources import DictSource @@ -334,6 +335,52 @@ def test_a_node_cancelled_by_a_failure_commits_its_ephemeral_results( assert delta_rows_named(tmp_path / "ephemeral", "plus_one") >= records +class RaisingNodeEnd(NoOpObserver): + """An observer whose ``on_node_end`` raises for node ``dbl``.""" + + def on_node_end(self, node_label: str, node_hash: str) -> None: + """Raise for ``dbl``.""" + if node_label == "dbl": + raise RuntimeError("observer failed") + + +def test_a_failing_observer_does_not_skip_the_failed_nodes_commit( + tmp_path: Path, +) -> None: + """An observer raising in ``on_node_end`` as a node fails still lets the node commit. + + The node's failure under ``fail_fast`` ends it for the observer and then commits + its ephemeral results; the observer's own exception must not skip that commit, as + on the sync path, where it propagates to the handler that commits. + """ + job = PipelineJob(name="batched", database=DeltaTableDatabase(tmp_path / "store")) + with job: + source = DictSource( + data=[{"k": i, "x": i} for i in range(CANCEL_ROWS)], + tag_columns=["k"], + source_id="src", + ) + pod = FunctionPod.from_fn( + double_until_last, + output_keys="y", + pod_config=PodConfig(max_concurrency=1), + ) + pod(source, label="dbl") + job.set_ephemeral_store(DeltaTableDatabase(tmp_path / "ephemeral")) + job.apply_node_config(NodeConfig(is_result_ephemeral=True)) + + with pytest.raises(RuntimeError, match="observer failed"): + job.run( + orchestrator=AsyncPipelineOrchestrator(error_policy="fail_fast"), + observer=RaisingNodeEnd(), + commit_every_rows=100, + ) + + records = delta_rows_named(tmp_path / "store", "double_until_last") + assert records == CANCEL_ROWS - 1 + assert delta_rows_named(tmp_path / "ephemeral", "double_until_last") >= records + + class FailPipelineRecords: """A store whose pipeline-record writes fail after the first ``ok``.""" From dbf8843918016e34911fef9315c945cacb63208d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 01:45:09 -0800 Subject: [PATCH 6/7] fix(function_node): a failed or cancelled async node holds no rows; on_node_end is called once (ITL-879) A node the async run cancels after another node's failure now reads as a node with no run (ITL-1093), also off the failing node's lineage. An observer whose on_node_end raises at the end of an async run is no longer called a second time by the failure branch. Tests expecting an ExceptionGroup from an async run now expect the node's own error. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-879.md | 2 +- design_issues/ITL-879-EP1.md | 8 +- docs/concepts/function-pods.md | 4 +- src/orcapod/core/nodes/function_node.py | 9 +- .../test_side_effect_job_node_reads.py | 2 +- .../test_pipeline/test_async_error_policy.py | 107 ++++++++++++++++++ .../test_job_node_rows_after_run.py | 4 +- .../test_operator_node_async_readback.py | 8 +- 8 files changed, 126 insertions(+), 18 deletions(-) diff --git a/changelog.d/ITL-879.md b/changelog.d/ITL-879.md index 64a1a1abf..0d18d2d04 100644 --- a/changelog.d/ITL-879.md +++ b/changelog.d/ITL-879.md @@ -7,4 +7,4 @@ ### Fixed - Under the async orchestrator a function node whose run stops on a row failure now calls the observer's `on_node_end` before raising, as the sync orchestrator does (ITL-879). -- A function node that the async orchestrator cancels because another node failed now commits the results it wrote before the cancellation propagates, as a failing node does (ITL-879). Before, its ephemeral results were never committed while the job committed its pipeline records, and on a Delta store its results were not committed ahead of its pipeline records. +- A function node that the async orchestrator cancels because another node failed now commits the results it wrote before the cancellation propagates, as a failing node does, and then holds no rows: it reads as a node with no run, as a node whose run raised (ITL-879, ITL-1093). Before, its ephemeral results were never committed while the job committed its pipeline records, and on a Delta store its results were not committed ahead of its pipeline records. diff --git a/design_issues/ITL-879-EP1.md b/design_issues/ITL-879-EP1.md index c450fda40..3f10abac7 100644 --- a/design_issues/ITL-879-EP1.md +++ b/design_issues/ITL-879-EP1.md @@ -29,9 +29,11 @@ in topological order (ties: the order the failures happened), the others noted o (`_single_failure`). A function node cancelled by another node's failure commits what it wrote (`_commit_after_failure`) before the cancellation propagates, as a failing node does: the job's flush commits neither its ephemeral store nor its result table ahead of its -pipeline records (CodeRabbit, PR #379). An observer whose `on_node_end` raises as the node -fails does not skip that commit either, as on the sync path. Rows run concurrently, so the row raised is the first to fail, the first -in input order only with `max_concurrency=1`. Storage guarantee, as documented in +pipeline records (CodeRabbit, PR #379); it then holds no rows, as a node whose run raised +(ITL-1093). An observer whose `on_node_end` raises as the node fails does not skip that +commit either, as on the sync path, and is not called a second time. Rows run concurrently, +so the row raised is the first to fail, the first in input order only with +`max_concurrency=1`. Storage guarantee, as documented in `docs/concepts/function-pods.md`: every stored row is a correct, complete result of its input row; the failing row is never stored; which other rows are stored is not specified. No hash, golden value or stored layout changed. Regressions: diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 272aead79..c48580d1e 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -568,7 +568,9 @@ has finished. When rows of several nodes fail before the run stops, one error is the others are noted on it (`error.__notes__`): a store error first (see "When the store cannot be written" below), then a `TypeIdentityError`, and otherwise the error of the node that comes first in the pipeline. A node cancelled by the failure commits the results it -wrote before it stops. +wrote before it stops. A run that raised defines no rows: afterwards the failing node, and +every node the run cancelled, reads as a node with no finished run, every row stored under +its key. What is stored after such a run: diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 3915ff325..ab84ed88c 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -3445,13 +3445,14 @@ async def async_execute( obs = observer if observer is not None else NoOpObserver() ctx_obs = obs.contextualize(*self.node_identity_path) - node_started = False + # Whether the node has started for the observer and not yet ended. + node_open = False try: tag_schema = self._input_stream.output_schema( columns={"system_tags": True} )[0] ctx_obs.on_node_start(node_label, node_hash, tag_schema=tag_schema) - node_started = True + node_open = True if self._cached_function_pod is not None: # ---------------------------------------------------------- @@ -3695,6 +3696,8 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: run_id=run_id, ) + # Ended for the observer here, also when the call raises: not again below. + node_open = False ctx_obs.on_node_end(node_label, node_hash) # The run is complete: stamp the node as the sync execute() does, before # the finally closes the output, so every downstream node, which stamps @@ -3710,7 +3713,7 @@ def create_data_logger(self, tag: Any, data: Any) -> Any: raise except Exception as run_error: try: - if node_started: + if node_open: # The node ends for the observer before it raises, as in execute; # a failing observer must not replace a store error on its way out. with keep_store_error( diff --git a/tests/test_core/side_effect_pod/test_side_effect_job_node_reads.py b/tests/test_core/side_effect_pod/test_side_effect_job_node_reads.py index 47e803a67..2411ff8d9 100644 --- a/tests/test_core/side_effect_pod/test_side_effect_job_node_reads.py +++ b/tests/test_core/side_effect_pod/test_side_effect_job_node_reads.py @@ -274,7 +274,7 @@ def test_rows_whose_delivery_failed_keep_todays_row_set( def test_a_run_that_raises_leaves_no_rows_to_read(orchestrator: Orchestrator) -> None: """After a run that raised, a read returns no rows and delivers nothing.""" job, sent = _job(failing={3}) - with pytest.raises(BaseException): # noqa: B017 # sync raises RuntimeError, async an ExceptionGroup + with pytest.raises(RuntimeError): _run(job, orchestrator) count = len(sent) node = job.nodes["delivered"] diff --git a/tests/test_pipeline/test_async_error_policy.py b/tests/test_pipeline/test_async_error_policy.py index 0fc2b24bd..2d9a6ab20 100644 --- a/tests/test_pipeline/test_async_error_policy.py +++ b/tests/test_pipeline/test_async_error_policy.py @@ -342,6 +342,113 @@ def test_max_concurrency_one_raises_the_first_failing_row(orchestrator: str) -> run_job(job, orchestrator, "fail_fast") +def rows_of(table: Any) -> list[tuple[tuple[str, Any], ...]]: + """The rows of ``table``, each a sorted tuple of (column, value), sorted.""" + return sorted(tuple(sorted(row.items())) for row in table.to_pylist()) + + +@pytest.mark.parametrize("position", list(FAILING_POSITIONS)) +def test_fail_fast_leaves_the_nodes_with_no_run( + orchestrator: str, build: Callable[..., PipelineJob], position: str +) -> None: + """After a fail_fast raise, the failing node and the node after it hold no run. + + A run that raised defines no rows (ITL-1093): each node reads as a node with no run, + every row stored under its content hash, also the downstream node the async run + cancelled. Another job stores rows of other values first, so a node still holding + its previous run's rows reads fewer rows than a node with no run. + """ + store = InMemoryArrowDatabase() + run_job(build(store, [100 + i for i in range(N_ROWS)]), orchestrator, "continue") + xs = xs_failing_at(FAILING_POSITIONS[position]) + job = build(store, xs) + run_job(job, orchestrator, "continue") + for label in ("f", "down"): + assert len(job.nodes[label].as_table()) == N_ROWS - 1, label + + with pytest.raises(ValueError): + run_job(job, orchestrator, "fail_fast") + + for label in ("f", "down"): + no_run = rows_of(build(store, xs).nodes[label].as_table()) + assert rows_of(job.nodes[label].as_table()) == no_run, label + # Also the other job's rows: not the previous run's rows only. + assert len(no_run) == 2 * N_ROWS - 1, label + + +# The row ``sibling`` leaves uncomputed until the run that is cancelled. +SIBLING_X = 2 +sibling_mode = {"mode": "fail"} + + +@function_pod(output_keys="w") +async def sibling(x: int) -> int: + """Return ``x``; at ``SIBLING_X`` fail, or wait until cancelled, as ``sibling_mode`` says.""" + if x == SIBLING_X: + if sibling_mode["mode"] == "fail": + raise ValueError("sibling failed") + await asyncio.Event().wait() + return x + + +def branches_job(store: Any, xs: list[int]) -> PipelineJob: + """Source -> ``f`` (fails on a negative x), and source -> ``g`` (``sibling``).""" + job = PipelineJob(name="itl879", database=store) + with job: + source = source_of(xs) + checked.pod(source, label="f") + sibling.pod(source, label="g") + return job + + +def test_a_node_cancelled_on_another_branch_holds_no_run() -> None: + """A node the async run cancels, not downstream of the failure, holds no run either. + + ``g`` waits at its one uncomputed row until the failure of ``f`` cancels it. Its + previous run's rows are not held: it reads as a node with no run (ITL-1093). + """ + store = InMemoryArrowDatabase() + other = [100 + i for i in range(N_ROWS)] + run_job(branches_job(store, other), "async", "continue") + xs = xs_failing_at(N_ROWS - 1) + job = branches_job(store, xs) + sibling_mode["mode"] = "fail" + run_job(job, "async", "continue") + assert len(job.nodes["g"].as_table()) == N_ROWS - 1 + + sibling_mode["mode"] = "wait" + try: + with pytest.raises(ValueError, match=r"^bad x="): + run_job(job, "async", "fail_fast") + finally: + sibling_mode["mode"] = "fail" + + no_run = rows_of(branches_job(store, xs).nodes["g"].as_table()) + assert rows_of(job.nodes["g"].as_table()) == no_run + assert len(no_run) == 2 * N_ROWS - 1 + + +def test_an_observer_failing_at_node_end_is_called_once(orchestrator: str) -> None: + """An ``on_node_end`` that raises is not called again; its error is the run's.""" + + class FailsAtNodeEnd(Recorder): + """A recorder whose ``on_node_end`` raises for node ``f``.""" + + def on_node_end(self, node_label: str, node_hash: str) -> None: + """Record the node end, then raise for ``f``.""" + super().on_node_end(node_label, node_hash) + if node_label == "f": + raise RuntimeError("observer failed") + + recorder = FailsAtNodeEnd() + job = function_job(InMemoryArrowDatabase(), [1, 2, 3]) + with pytest.raises(RuntimeError, match="^observer failed$") as info: + run_job(job, orchestrator, "continue", recorder) + + assert type(info.value) is RuntimeError + assert recorder.of("node_end", "f") == [None] + + def test_nothing_left_running_after_fail_fast( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/test_pipeline/test_job_node_rows_after_run.py b/tests/test_pipeline/test_job_node_rows_after_run.py index c31e7c487..defb0b1f5 100644 --- a/tests/test_pipeline/test_job_node_rows_after_run.py +++ b/tests/test_pipeline/test_job_node_rows_after_run.py @@ -306,6 +306,7 @@ def _no_run_read(fn: Any, data: list[dict[str, int]], db: Any) -> list[Any]: ("kind", "fn", "policy"), [ ("sync", fails_on_20, "fail_fast"), + ("async", fails_on_20, "fail_fast"), ("sync", store_fails_on_20, "continue"), ("async", store_fails_on_20, "continue"), ], @@ -470,8 +471,7 @@ def on_node_end(self, node_label: str, node_hash: str) -> None: job.run(orchestrator=_orchestrator(kind)) node = job.nodes["node"] assert _values(node, lambda r: r["k"]) == [0, 1, 2] - # The async orchestrator raises its task group's error around it. - with pytest.raises((RuntimeError, ExceptionGroup)): + with pytest.raises(RuntimeError, match="^observer failed$"): job.run(orchestrator=_orchestrator(kind), observer=FailsAtNodeEnd()) # Every stored row (the 6-row run's), read from the store. assert _values(node, lambda r: r["k"]) == [0, 1, 2, 3, 4, 5] diff --git a/tests/test_pipeline/test_operator_node_async_readback.py b/tests/test_pipeline/test_operator_node_async_readback.py index acc72a688..3dee317cc 100644 --- a/tests/test_pipeline/test_operator_node_async_readback.py +++ b/tests/test_pipeline/test_operator_node_async_readback.py @@ -713,14 +713,8 @@ def test_join_with_an_input_that_delivered_no_row_fails_alike_on_both_paths( lambda a, b, c: Join()(_no_match(a), b, label="op"), 0, ) - with pytest.raises((InputValidationError, ExceptionGroup)) as info: + with pytest.raises(InputValidationError, match="'id'"): _ = _run_job(case, orchestrator, "in_memory", tmp_path) - error = info.value - while isinstance(error, ExceptionGroup): - assert len(error.exceptions) == 1 - error = error.exceptions[0] - assert isinstance(error, InputValidationError) - assert "'id'" in str(error) def test_rebuilt_upstream_keeps_the_upstream_data_context() -> None: From 92267862e39c0e11c8b6546bcd9cf599a38aaa15 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Sat, 14 Nov 2026 02:07:25 -0800 Subject: [PATCH 7/7] test(async_error_policy): an aggregate after a fail_fast failure stores nothing; helper docstrings (ITL-879) A Batch node in CacheMode.LOG downstream of the failing node stores no aggregate after an async fail_fast raise (failure first, middle, last; max_concurrency 1 and unlimited), and stores its one aggregate under continue. Docstrings for the test helpers that had none (CodeRabbit). Co-Authored-By: Claude Opus 5.5 --- .../test_pipeline/test_async_error_policy.py | 51 ++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/tests/test_pipeline/test_async_error_policy.py b/tests/test_pipeline/test_async_error_policy.py index 2d9a6ab20..5d8cb2d13 100644 --- a/tests/test_pipeline/test_async_error_policy.py +++ b/tests/test_pipeline/test_async_error_policy.py @@ -22,6 +22,7 @@ from orcapod import PipelineJob, function_pod from orcapod.core.function_pod import FunctionPod +from orcapod.core.operators import Batch from orcapod.databases import InMemoryArrowDatabase from orcapod.errors import RowFailuresWarning from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator @@ -29,7 +30,7 @@ from orcapod.pipeline.observer import NoOpObserver from orcapod.side_effects import InvocationContext, SideEffectPod, SideEffectPodConfig from orcapod.sources import DictSource -from orcapod.types import PodConfig +from orcapod.types import CacheMode, PodConfig RUN_TIMEOUT_SECONDS = 60 N_ROWS = 6 @@ -188,6 +189,7 @@ def run_job( return job.run(orchestrator=orch, observer=observer) async def guarded() -> Any: + """Run the job, failing after ``RUN_TIMEOUT_SECONDS`` instead of hanging.""" async with asyncio.timeout(RUN_TIMEOUT_SECONDS): return await job.run_async(orchestrator=orch, observer=observer) @@ -457,12 +459,18 @@ def test_nothing_left_running_after_fail_fast( # The orchestrator's channels are internal; record them through its module's names. class RecordingChannel(async_orchestrator_module.Channel[Any]): + """A ``Channel`` that records itself in ``created``.""" + def __post_init__(self) -> None: + """Set up the channel, then record it.""" super().__post_init__() created.append(self) class RecordingBroadcastChannel(async_orchestrator_module.BroadcastChannel[Any]): + """A ``BroadcastChannel`` that records itself in ``created``.""" + def __init__(self, buffer_size: int = 64) -> None: + """Set up the channel with ``buffer_size``, then record it.""" super().__init__(buffer_size) created.append(self) @@ -477,6 +485,7 @@ def __init__(self, buffer_size: int = 64) -> None: triple.pod(out, label="down2") # fan-out: a broadcast channel async def main() -> set[asyncio.Task[Any]]: + """Run the job under a timeout guard; return the tasks still pending after it.""" async with asyncio.timeout(RUN_TIMEOUT_SECONDS): with pytest.raises(ValueError): await job.run_async( @@ -492,6 +501,45 @@ async def main() -> set[asyncio.Task[Any]]: assert closed.is_set() if isinstance(closed, asyncio.Event) else closed +def stored_aggregates( + job_store: Any, policy: str, xs: list[int], mc: int | None +) -> int: + """Run source -> ``f`` -> ``Batch`` node ``B`` (LOG mode) async; count ``B``'s stored rows.""" + log = InMemoryArrowDatabase() + job = function_job(job_store, xs, max_concurrency=mc) + with job: + Batch()(job.nodes["f"], label="B") + job.nodes["B"].attach_databases(pipeline_database=log, cache_mode=CacheMode.LOG) + if policy == "fail_fast": + with pytest.raises(ValueError): + run_job(job, "async", policy) + else: + run_job(job, "async", policy) + tables = [log.get_all_records(path) for path in log.list_table_paths()] + return sum(table.num_rows for table in tables if table is not None) + + +@pytest.mark.parametrize("max_concurrency", [1, None]) +@pytest.mark.parametrize("position", list(FAILING_POSITIONS)) +def test_an_aggregate_downstream_stores_nothing_after_fail_fast( + max_concurrency: int | None, position: str +) -> None: + """A ``Batch`` node after the failing node never stores an aggregate of partial input. + + The failing node closes its output before the run cancels the others; the aggregate + must not take that close for the end of its input. Under ``continue`` the same job + stores its one aggregate, so the store is read where the aggregate would be. + """ + xs = xs_failing_at(FAILING_POSITIONS[position]) + assert ( + stored_aggregates(InMemoryArrowDatabase(), "continue", xs, max_concurrency) == 1 + ) + assert ( + stored_aggregates(InMemoryArrowDatabase(), "fail_fast", xs, max_concurrency) + == 0 + ) + + # --------------------------------------------------------------------------- # Side-effect pods: their own on_error, not error_policy # --------------------------------------------------------------------------- @@ -535,6 +583,7 @@ class _Node: """A stand-in node: only its label is read.""" def __init__(self, label: str) -> None: + """Name the node ``label``.""" self.label = label