From a54ba4672bc7b58cb7d0ab73ac2e42bca0f8514f Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 16:09:59 -0800 Subject: [PATCH 1/6] test(pipeline): WIP failing tests for a job run in a running loop and a run summary (ITL-994, ITL-997) Co-Authored-By: Claude Opus 5.5 --- .../plans/2026-11-12-itl-994-997-job-run.md | 107 ++++++++++ .../test_job_run_in_event_loop.py | 136 +++++++++++++ tests/test_pipeline/test_job_run_summary.py | 184 ++++++++++++++++++ 3 files changed, 427 insertions(+) create mode 100644 superpowers/plans/2026-11-12-itl-994-997-job-run.md create mode 100644 tests/test_pipeline/test_job_run_in_event_loop.py create mode 100644 tests/test_pipeline/test_job_run_summary.py diff --git a/superpowers/plans/2026-11-12-itl-994-997-job-run.md b/superpowers/plans/2026-11-12-itl-994-997-job-run.md new file mode 100644 index 000000000..a428d68a2 --- /dev/null +++ b/superpowers/plans/2026-11-12-itl-994-997-job-run.md @@ -0,0 +1,107 @@ +# ITL-994 + ITL-997: running a job in a notebook, and reporting failed rows + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this +> plan task by task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** `PipelineJob.run` works with the async orchestrator inside a running event loop +(Jupyter), and after every run the user sees which rows failed, in which node, and why. + +**Architecture:** ITL-994: `AsyncPipelineOrchestrator.run` runs its coroutine on a bridge +thread with its own loop when the caller's thread already runs a loop; `PipelineJob.run_async` +awaits the orchestrator's `run_async` for async code. ITL-997: `job.run` wraps the user's +observer with a private counting observer and returns a `RunSummary` built from its counts; +a `RowFailuresWarning` is issued when any row failed; `job.last_run` keeps the summary. + +**Tech stack:** asyncio, threading, warnings; existing `CompositeObserver`. + +**Branch:** `eywalker/itl-994-a-pipelinejob-cannot-be-run-with-the-async` from origin/main +(290889e8). One PR: Fixes ITL-994, Fixes ITL-997; one commit per issue; one changelog +fragment per issue. + +--- + +## Status / notes for a successor + +- 2026-11-12: investigation done, design sent to the coordinator (ask). Coordinator: docs go in + `docs/concepts/function-pods.md`; the public-interface choices went to the developer. + Waiting for: 994 option (recommended (c): run bridges + run_async), 997 R1+R2+R3 + (recommended), per-row sync log to DEBUG (recommended), names. +- Allowed meanwhile: decision-independent failing tests, with the entry points and names kept + in one place at the top of each test module. + +## Findings (today, main 290889e8) + +- `AsyncPipelineOrchestrator.run` = `asyncio.run(self._run_async(...))` -> RuntimeError in + any running loop. `run_async` exists on the orchestrator, not on the job. +- `job.run()` returns `self`; ~38 test sites use the return value; no src callers. +- Sync: per-row `logger.warning(..., exc_info=True)` (function_node.py ~1763) prints one full + traceback per failed row. Async: `logger.debug` (function_pod.py ~619): nothing shown. +- Observer per function-node row, both orchestrators: `on_data_end(cached=True)` (from the + store), `on_data_end(cached=False)` (computed; output None = filtered), `on_data_crash` + (with the exception object). + +## Files + +- Modify `src/orcapod/pipeline/async_orchestrator.py`: `run` bridges when a loop is running. +- Modify `src/orcapod/pipeline/job.py`: split `run` into `_prepare_run` / `_finish_run`; + add `run_async`; counting observer; return `RunSummary`; `last_run`. +- Create `src/orcapod/pipeline/run_summary.py`: `RunSummary`, `NodeRunSummary`, + `_RunCountingObserver`. +- Modify `src/orcapod/errors.py`: `RowFailuresWarning(UserWarning)`. +- Modify `src/orcapod/core/nodes/function_node.py`: per-row failure log level (if approved). +- Tests: `tests/test_pipeline/test_job_run_in_event_loop.py` (994), + `tests/test_pipeline/test_job_run_summary.py` (997); update callers of `job.run()`'s + return value. +- Docs: `docs/concepts/function-pods.md` ("Running a job": the async run in a notebook and a + script; reading the run summary). +- `changelog.d/ITL-994.md`, `changelog.d/ITL-997.md`. + +## Task 1 (ITL-994): failing tests + +- [ ] `test_job_run_in_event_loop.py`: a job run with the async orchestrator from inside a + running loop equals a run outside one (stored records minus per-run columns `__data_id`, + `__pod_ts`; node table with all info), InMemory and Delta; `await job.run_async()` the same; + a store error inside the bridged run reaches the caller with its type; KeyboardInterrupt + while waiting cancels the run and re-raises. +- [ ] Run, see them fail with "asyncio.run() cannot be called from a running event loop". + +## Task 2 (ITL-994): implementation + +- [ ] `AsyncPipelineOrchestrator.run`: no running loop -> `asyncio.run` as today. Running loop + -> new thread, `contextvars.copy_context().run(...)`, own loop, run `_run_async` as a task; + caller waits on the thread; on KeyboardInterrupt cancel the task with + `loop.call_soon_threadsafe(task.cancel)`, join, re-raise KeyboardInterrupt; other exceptions + re-raised in the caller unchanged. +- [ ] `PipelineJob.run_async(orchestrator=None, observer=None, materialize_results=True, + commit_every_rows=100)`: default `AsyncPipelineOrchestrator()`; a sync orchestrator -> + `TypeError` naming `job.run`. Shares `_prepare_run` / `_finish_run` with `run`. +- [ ] Manual check in a real ipykernel (jupyter_client): both forms; interrupt. +- [ ] changelog.d/ITL-994.md, docs, commit `fix(pipeline): run a job with the async + orchestrator inside a running event loop (ITL-994)`. + +## Task 3 (ITL-997): failing tests + +- [ ] `test_job_run_summary.py`, sync and async, InMemory and Delta: pod failing on some rows + and on all rows -> summary counts (computed / from store / failed), first error type and + message, node label; exactly one `RowFailuresWarning` with that text; clean run: no warning; + re-run: rows from store; `fail_fast` still raises; repr and `_repr_html_`; `print_traceback` + shows the function's frame; `job.last_run` is the returned object. + +## Task 4 (ITL-997): implementation + +- [ ] `_RunCountingObserver` (lock-protected counts per node label; first error per node), + composed with the user's observer through `CompositeObserver`. +- [ ] `RunSummary` / `NodeRunSummary` with `__repr__`, `_repr_html_`, `print_traceback`. +- [ ] `run` / `run_async` return the summary, set `job.last_run`, warn once when rows failed. +- [ ] Update the ~38 test sites that use `job.run()`'s return value as the job. +- [ ] Per-row sync failure log to DEBUG (if approved). +- [ ] changelog.d/ITL-997.md, docs, commit `feat(pipeline)!: job.run returns a run summary + and warns when rows failed (ITL-997)`. + +## Task 5: finish + +- [ ] ruff check/format, basedpyright exit code, full `uv run pytest tests/`, golden tests + unchanged, `scripts/assemble_changelog.py --check`. +- [ ] Merge origin/main, push, PR (create-pr skill), `orca linear attach`, + `orca worktree set --worktree current --pr `, CodeRabbit rounds (inline and review + bodies), CI green, worker_done. diff --git a/tests/test_pipeline/test_job_run_in_event_loop.py b/tests/test_pipeline/test_job_run_in_event_loop.py new file mode 100644 index 000000000..7f533e503 --- /dev/null +++ b/tests/test_pipeline/test_job_run_in_event_loop.py @@ -0,0 +1,136 @@ +"""A job runs with the async orchestrator from inside a running event loop (ITL-994). + +A Jupyter kernel always runs an event loop, so ``asyncio.run`` cannot be called there. A job +run from inside a running loop must work, and must store exactly what a run outside a loop +stores: the same rows, record ids and hashes. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pyarrow as pa +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator +from orcapod.sources import DictSource + +# Columns that differ between any two runs by design: the datagram uuid and the time the +# result was computed. +PER_RUN_COLUMNS = ("__data_id", "__pod_ts") + +STORES: dict[str, Callable[[Path], Any]] = { + "in_memory": lambda _: InMemoryArrowDatabase(), + "delta": lambda path: DeltaTableDatabase(path), +} + + +def run_inside_running_loop(job: PipelineJob) -> None: + """Run ``job`` with the async orchestrator from inside a running event loop. + + The one place that names the call an attendee types in a notebook cell. + """ + + async def notebook_cell() -> None: + job.run(orchestrator=AsyncPipelineOrchestrator()) + + asyncio.run(notebook_cell()) + + +@function_pod(output_keys="y") +def double(x: int) -> int: + """Return twice ``x``.""" + return 2 * x + + +def make_job(store: Any) -> PipelineJob: + """A source of five rows feeding ``double``, recorded in a fresh job.""" + rows = [{"k": f"r{i}", "x": i} for i in range(5)] + job = PipelineJob(name="loop_demo", store=store) + with job: + source = DictSource(data=rows, tag_columns=["k"], source_id="rows") + double.pod(source, label="doubled") + return job + + +def stored_rows(job: PipelineJob) -> list[dict[str, Any]]: + """The node's stored records and its table, minus per-run columns, sorted by tag.""" + records = job.nodes["doubled"].get_all_records(all_info=True) + assert records is not None + records = records.drop_columns( + [c for c in PER_RUN_COLUMNS if c in records.column_names] + ) + table = job.nodes["doubled"].as_table(all_info=True) + joined = records.join(table, keys="k", join_type="inner", right_suffix="_table") + return sorted(joined.to_pylist(), key=lambda row: row["k"]) + + +@pytest.mark.parametrize("store_kind", list(STORES)) +def test_run_inside_running_loop_stores_what_a_script_run_stores( + store_kind: str, tmp_path: Path +) -> None: + script_job = make_job(STORES[store_kind](tmp_path / "script")) + script_job.run(orchestrator=AsyncPipelineOrchestrator()) + + notebook_job = make_job(STORES[store_kind](tmp_path / "notebook")) + run_inside_running_loop(notebook_job) + + assert stored_rows(notebook_job) == stored_rows(script_job) + ys = sorted(notebook_job.nodes["doubled"].as_table().column("y").to_pylist()) + assert ys == [0, 2, 4, 6, 8] + + +def test_rerun_inside_running_loop_reuses_stored_rows(tmp_path: Path) -> None: + store = DeltaTableDatabase(tmp_path / "store") + first_job = make_job(store) + run_inside_running_loop(first_job) + first = stored_rows(first_job) + + second_job = make_job(store) + run_inside_running_loop(second_job) + + assert stored_rows(second_job) == first + + +def test_store_error_inside_running_loop_reaches_the_caller(tmp_path: Path) -> None: + """A failure inside the run reaches the cell with its own type, not wrapped.""" + from orcapod.errors import StoreWriteError + + class FailingWrites: + """A store whose every write fails.""" + + def __init__(self, inner: Any) -> None: + self._inner = inner + + def at(self, *path_components: str) -> FailingWrites: + return FailingWrites(self._inner.at(*path_components)) + + def add_record(self, *args: Any, **kwargs: Any) -> Any: + raise OSError("disk unavailable") + + def add_records(self, *args: Any, **kwargs: Any) -> Any: + raise OSError("disk unavailable") + + def __getattr__(self, name: str) -> Any: + return getattr(self._inner, name) + + job = make_job(FailingWrites(InMemoryArrowDatabase())) + with pytest.raises(StoreWriteError): + run_inside_running_loop(job) + + +def test_run_inside_running_loop_returns_to_the_loop(tmp_path: Path) -> None: + """The loop that called the run keeps working after it (a notebook's next cell).""" + + async def two_cells() -> int: + job = make_job(InMemoryArrowDatabase()) + job.run(orchestrator=AsyncPipelineOrchestrator()) + await asyncio.sleep(0) + return pa.table({"n": [1]}).num_rows + + assert asyncio.run(two_cells()) == 1 diff --git a/tests/test_pipeline/test_job_run_summary.py b/tests/test_pipeline/test_job_run_summary.py new file mode 100644 index 000000000..9843763aa --- /dev/null +++ b/tests/test_pipeline/test_job_run_summary.py @@ -0,0 +1,184 @@ +"""After a job run, the user is told which rows failed (ITL-997). + +With the default ``error_policy="continue"`` a row whose function raises is skipped and the +run goes on. The user must still see, without configuring anything: the node, how many rows +failed out of how many, and the first error's type and message, with a pointer to the full +traceback. A clean run stays quiet. +""" + +from __future__ import annotations + +import warnings +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase +from orcapod.errors import RowFailuresWarning +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.sources import DictSource + +# --- The public names this feature adds, in one place --------------------------------- + + +def run(job: PipelineJob, orchestrator: Any) -> Any: + """Run ``job``; return what the user gets back (the run summary).""" + return job.run(orchestrator=orchestrator) + + +def node_counts(summary: Any, label: str) -> tuple[int, int, int]: + """``(computed, from store, failed)`` rows of node ``label`` in ``summary``.""" + node = summary.nodes[label] + return node.computed, node.reused, node.failed + + +def first_error(summary: Any, label: str) -> BaseException | None: + """The first error raised in node ``label`` during the run.""" + return summary.nodes[label].first_error + + +def last_run(job: PipelineJob) -> Any: + """The summary of the job's last completed run.""" + return job.last_run + + +# -------------------------------------------------------------------------------------- + +ORCHESTRATORS: dict[str, Callable[[], Any]] = { + "sync": SyncPipelineOrchestrator, + "async": AsyncPipelineOrchestrator, +} + +STORES: dict[str, Callable[[Path], Any]] = { + "in_memory": lambda _: InMemoryArrowDatabase(), + "delta": lambda path: DeltaTableDatabase(path), +} + + +@function_pod(output_keys="y") +def even_only(x: int) -> int: + """Return ``x``; raise for an odd ``x``.""" + if x % 2: + raise ValueError(f"odd x={x}") + return x + + +@function_pod(output_keys="y") +def always_fails(x: int) -> int: + """Raise for every row, as a function with a typo does.""" + return x / undefined_name # noqa: F821 # pyright: ignore[reportUndefinedVariable] # the bug under test + + +@function_pod(output_keys="y") +def double(x: int) -> int: + """Return twice ``x``.""" + return 2 * x + + +def make_job(store: Any, pod: Any, label: str, n_rows: int = 4) -> PipelineJob: + """A source of ``n_rows`` rows (x = 0..n-1) feeding ``pod`` under ``label``.""" + rows = [{"k": f"r{i}", "x": i} for i in range(n_rows)] + job = PipelineJob(name="summary_demo", store=store) + with job: + source = DictSource(data=rows, tag_columns=["k"], source_id="rows") + pod.pod(source, label=label) + return job + + +@pytest.fixture(params=list(ORCHESTRATORS)) +def orchestrator(request: pytest.FixtureRequest) -> Any: + return ORCHESTRATORS[request.param]() + + +@pytest.fixture(params=list(STORES)) +def store(request: pytest.FixtureRequest, tmp_path: Path) -> Any: + return STORES[request.param](tmp_path / "store") + + +def test_some_rows_fail(store: Any, orchestrator: Any) -> None: + job = make_job(store, even_only, "evens") + with pytest.warns(RowFailuresWarning) as caught: + summary = run(job, orchestrator) + + assert node_counts(summary, "evens") == (2, 0, 2) + error = first_error(summary, "evens") + assert isinstance(error, ValueError) + assert str(error).startswith("odd x=") + assert len(caught) == 1 + message = str(caught[0].message) + assert "2 of 4 rows failed in node 'evens'" in message + assert "ValueError: odd x=" in message + assert 'job.last_run.print_traceback("evens")' in message + assert last_run(job) is summary + # The rows that succeeded were computed and stored as before. + ys = sorted(job.nodes["evens"].as_table().column("y").to_pylist()) + assert ys == [0, 2] + + +def test_every_row_fails(store: Any, orchestrator: Any) -> None: + job = make_job(store, always_fails, "broken") + with pytest.warns(RowFailuresWarning, match="4 of 4 rows failed in node 'broken'"): + summary = run(job, orchestrator) + + assert node_counts(summary, "broken") == (0, 0, 4) + assert isinstance(first_error(summary, "broken"), NameError) + assert summary.failed_rows == 4 + assert not summary.ok + text = repr(summary) + assert "broken" in text + assert "NameError: name 'undefined_name' is not defined" in text + assert 'job.last_run.print_traceback("broken")' in text + + +def test_clean_run_is_quiet(store: Any, orchestrator: Any) -> None: + job = make_job(store, double, "doubled") + with warnings.catch_warnings(): + warnings.simplefilter("error") + summary = run(job, orchestrator) + + assert summary.ok + assert summary.failed_rows == 0 + assert node_counts(summary, "doubled") == (4, 0, 0) + assert first_error(summary, "doubled") is None + + +def test_rerun_counts_rows_from_the_store(store: Any, orchestrator: Any) -> None: + run(make_job(store, double, "doubled"), orchestrator) + summary = run(make_job(store, double, "doubled", n_rows=5), orchestrator) + + assert node_counts(summary, "doubled") == (1, 4, 0) + + +def test_fail_fast_still_raises(store: Any) -> None: + job = make_job(store, even_only, "evens") + with pytest.raises(ValueError, match="odd x="): + job.run(orchestrator=SyncPipelineOrchestrator(error_policy="fail_fast")) + + +def test_print_traceback_shows_the_function_frame( + capsys: pytest.CaptureFixture[str], +) -> None: + job = make_job(InMemoryArrowDatabase(), always_fails, "broken") + with pytest.warns(RowFailuresWarning): + run(job, SyncPipelineOrchestrator()) + + last_run(job).print_traceback("broken") + + err = capsys.readouterr().err + assert "Traceback (most recent call last)" in err + assert "in always_fails" in err + assert "NameError: name 'undefined_name' is not defined" in err + + +def test_notebook_representation_names_the_failing_node() -> None: + job = make_job(InMemoryArrowDatabase(), even_only, "evens") + with pytest.warns(RowFailuresWarning): + summary = run(job, SyncPipelineOrchestrator()) + + html = summary._repr_html_() # the notebook display hook is what is under test + assert "evens" in html + assert "ValueError" in html + assert "odd x=" in html From 643357397263c1ba2d2b929ef2e68eb296884d9b Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 16:22:39 -0800 Subject: [PATCH 2/6] fix(pipeline): run a job with the async orchestrator inside a running event loop (ITL-994) job.run(orchestrator=AsyncPipelineOrchestrator()) raised 'asyncio.run() cannot be called from a running event loop' in a Jupyter kernel. AsyncPipelineOrchestrator.run now runs the same coroutine on a separate thread with its own loop when the caller's thread already runs one, and waits for it; an interrupt while waiting cancels the run and waits for it to stop. PipelineJob.run_async awaits the orchestrator on the caller's loop for async code. run's body is split into _prepare_run, _executing and _finish_run, shared by both forms. Stored rows, record ids and hashes are unchanged. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-994.md | 16 ++ docs/concepts/function-pods.md | 53 ++++++ src/orcapod/pipeline/async_orchestrator.py | 109 +++++++++++- src/orcapod/pipeline/job.py | 164 ++++++++++++++++-- .../plans/2026-11-12-itl-994-997-job-run.md | 18 +- .../test_job_run_in_event_loop.py | 87 +++++++++- 6 files changed, 416 insertions(+), 31 deletions(-) create mode 100644 changelog.d/ITL-994.md diff --git a/changelog.d/ITL-994.md b/changelog.d/ITL-994.md new file mode 100644 index 000000000..9ebba2215 --- /dev/null +++ b/changelog.d/ITL-994.md @@ -0,0 +1,16 @@ +### Added + +- **`PipelineJob.run_async`** — `await job.run_async()` runs a job with the + `AsyncPipelineOrchestrator` (its default) on the caller's event loop, for async code and + notebook cells. It takes the same arguments as `run`; a `SyncPipelineOrchestrator` raises + `TypeError`. (ITL-994) + +### Fixed + +- **`job.run(orchestrator=AsyncPipelineOrchestrator())` works inside a running event loop**, + such as a Jupyter kernel or async code. It used to raise + `RuntimeError: asyncio.run() cannot be called from a running event loop`. Inside a loop, the + run now goes on a separate thread while the call waits, so the same line works in a script + and in a notebook; it blocks that loop until the run ends (use `await job.run_async()` in + async code). The interrupt button stops the run and nothing keeps running after it. Stored + rows, record ids and hashes are the same as a run outside a loop. (ITL-994) diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index d9774df4e..dfa5f4c8a 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -398,6 +398,59 @@ print("function calls:", len(calls)) flush tries them again. A flush that fails for any other reason raises `RuntimeError`, with the first failure as its cause. +### Running asynchronously, in a script or a notebook + +`job.run()` uses the `SyncPipelineOrchestrator`, which runs one node after another. The +`AsyncPipelineOrchestrator` runs every node at once, passing rows between them as they are +ready, and runs your function on several rows concurrently. Choose it with the +`orchestrator` argument. The line is the same in a script and in a Jupyter notebook: + +```python +from orcapod.pipeline import AsyncPipelineOrchestrator + +job = PipelineJob(name="bmi_async", store=InMemoryArrowDatabase()) +with job: + source = DictSource(data=mice, tag_columns=["subject_id"], source_id="mice") + compute_bmi.pod(source, label="bmi") + +job.run(orchestrator=AsyncPipelineOrchestrator()) + +print(job.nodes["bmi"].as_table().to_pandas()) +# subject_id bmi +# 0 mouse_01 1756.944444 +# 1 mouse_02 2210.000000 +# 2 mouse_03 2297.520661 +``` + +Both orchestrators store the same rows, with the same record ids and hashes. + +A notebook kernel already runs an event loop. There, `job.run(...)` runs the job on a +separate thread and waits for it, so the cell blocks until the run ends, as it does with the +sync orchestrator. The interrupt button stops the run: rows already stored stay stored, and +the next run computes only the rest. + +In async code, await the run instead, so the event loop stays free while it runs: +`await job.run_async()` (the async orchestrator is its default). This also works at the top +level of a notebook cell. In a script, start async code with `asyncio.run`: + +```python +import asyncio + +job = PipelineJob(name="bmi_awaited", store=InMemoryArrowDatabase()) +with job: + source = DictSource(data=mice, tag_columns=["subject_id"], source_id="mice") + compute_bmi.pod(source, label="bmi") + + +async def main(): + await job.run_async() + + +asyncio.run(main()) +print(job.nodes["bmi"].as_table().num_rows) +# 3 +``` + ### When results are committed Inside `job.run()`, each function node commits its results and pipeline records to the diff --git a/src/orcapod/pipeline/async_orchestrator.py b/src/orcapod/pipeline/async_orchestrator.py index 16b85ecde..a6913c832 100644 --- a/src/orcapod/pipeline/async_orchestrator.py +++ b/src/orcapod/pipeline/async_orchestrator.py @@ -8,11 +8,14 @@ from __future__ import annotations import asyncio -import uuid +import contextlib +import contextvars import logging +import threading +import uuid from collections import defaultdict -from collections.abc import Coroutine -from typing import TYPE_CHECKING, Any, Literal +from collections.abc import Callable, Coroutine +from typing import TYPE_CHECKING, Any, Literal, TypeVar from orcapod.channels import BroadcastChannel, Channel from orcapod.errors import StoreError, StoreWriteError, TypeIdentityError @@ -33,6 +36,11 @@ logger = logging.getLogger(__name__) +_T = TypeVar("_T") + +# How often a caller waiting for a run on its own thread checks for an interrupt. +_INTERRUPT_CHECK_SECONDS = 0.1 + class AsyncPipelineOrchestrator: """Execute a compiled pipeline asynchronously using channels. @@ -69,6 +77,14 @@ def run( ) -> OrchestratorResult: """Synchronous entry point — runs the async pipeline to completion. + Without a running event loop on the calling thread (a script), the run uses + ``asyncio.run``. With one (a Jupyter kernel, or a call from async code), the + same coroutine runs to completion on a separate thread with its own event loop + while the caller waits, so the call works the same in both places. The caller's + loop is blocked until the run ends; async code should await ``run_async`` + instead. A ``KeyboardInterrupt`` while waiting (the notebook's interrupt button) + cancels the run, waits for it to stop, and is re-raised. + Args: graph: A ``GraphProtocol`` DAG with GraphNode objects as vertices. observer: Optional execution observer forwarded to nodes. @@ -83,15 +99,26 @@ def run( Returns: OrchestratorResult with node outputs. """ - return asyncio.run( - self._run_async( + + def make_run() -> Coroutine[Any, Any, OrchestratorResult]: + return self._run_async( graph, materialize_results, observer=observer, run_id=run_id, pipeline_uri=pipeline_uri, ) - ) + + try: + _ = asyncio.get_running_loop() + except RuntimeError: + loop_running = False + else: + loop_running = True + # Run outside the handler, so the run's own errors carry no context from it. + if loop_running: + return _run_on_own_thread(make_run) + return asyncio.run(make_run()) async def run_async( self, @@ -285,6 +312,76 @@ async def _run_async( effective_observer.on_run_end(run_id) +def _run_on_own_thread(make_run: Callable[[], Coroutine[Any, Any, _T]]) -> _T: + """Run a coroutine to completion on a new thread with its own event loop. + + For a caller whose thread already runs an event loop, where ``asyncio.run`` is + refused. The caller blocks until the run ends and gets its result, or its exception + unchanged. The thread runs in a copy of the caller's context variables. A + ``KeyboardInterrupt`` raised in the caller while it waits cancels the run, waits for + the run to stop (functions already running in the loop's thread pool finish their + current row), and is re-raised, so no run keeps writing after the caller stopped. + + Args: + make_run: Builds the coroutine; called on the new thread. + + Returns: + What the coroutine returned. + """ + loop = asyncio.new_event_loop() + tasks: list[asyncio.Task[_T]] = [] + task_created = threading.Event() + finished = threading.Event() + results: list[_T] = [] + errors: list[BaseException] = [] + + def run_loop() -> None: + asyncio.set_event_loop(loop) + try: + task = loop.create_task(make_run()) + tasks.append(task) + task_created.set() + results.append(loop.run_until_complete(task)) + except BaseException as exc: # noqa: BLE001 # handed to the waiting caller + errors.append(exc) + finally: + task_created.set() + try: + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.run_until_complete(loop.shutdown_default_executor()) + finally: + asyncio.set_event_loop(None) + loop.close() + finished.set() + + context = contextvars.copy_context() + thread = threading.Thread( + target=context.run, args=(run_loop,), name="orcapod-async-run", daemon=True + ) + thread.start() + # Wait on an event, not Thread.join: on Python 3.12 an interrupt inside join marks + # a thread that is still running as stopped, and later joins return at once. + try: + # Wait in short slices: a pending interrupt is raised between them even when + # the signal reached another thread and did not wake this one's wait. + while not finished.wait(_INTERRUPT_CHECK_SECONDS): + pass + except KeyboardInterrupt: + _ = task_created.wait() + if tasks: + # The loop may have closed after the run ended on its own. + with contextlib.suppress(RuntimeError): + _ = loop.call_soon_threadsafe(tasks[0].cancel) + # A second interrupt here stops the wait; the cancelled run then winds down + # on its own thread. + while not finished.wait(_INTERRUPT_CHECK_SECONDS): + pass + raise + if errors: + raise errors[0] + return results[0] + + def _start_node( tg: asyncio.TaskGroup, node: Any, work: Coroutine[Any, Any, None] ) -> None: diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 1a5e8a5cf..906ad1283 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -5,6 +5,7 @@ import contextlib import logging from collections.abc import Iterator +from dataclasses import dataclass from typing import TYPE_CHECKING, Any from orcapod.core.nodes import JobNode @@ -49,6 +50,25 @@ ) +@dataclass(frozen=True) +class _PreparedRun: + """What one run of a job needs, built by ``PipelineJob._prepare_run``. + + Attributes: + exec_dag: The nodes to run (unbound sources and their dependents left out). + run_id: The run's identifier. + pipeline_uri: ``@``, passed to the observer. + pipeline_db: The job's view of the store. + result_db: The job's function-result view of the store. + """ + + exec_dag: OrcaDAG[Any] + run_id: str + pipeline_uri: str + pipeline_db: "ArrowDatabaseProtocol" + result_db: "ArrowDatabaseProtocol" + + class PipelineJob(AbstractPipelineBase[JobNode]): """Pipeline + source bindings + execution context. @@ -785,6 +805,11 @@ def run( After a successful run, ``job.nodes`` returns the execution-ready nodes with populated database caches. + ``job.run(orchestrator=AsyncPipelineOrchestrator())`` works the same in a + script and inside a running event loop, such as a Jupyter kernel: there the + run goes on a separate thread while the call waits, and blocks that loop + until it ends. From async code, ``await job.run_async()`` instead. + Args: orchestrator: Orchestrator to use for execution. Defaults to ``SyncPipelineOrchestrator`` when ``None``. @@ -840,12 +865,93 @@ def run( was needed to write a node's results. Its rows are not recomputed and the table is not overwritten. """ - import hashlib - import uuid - from orcapod.pipeline.observer import NoOpObserver from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator + prepared = self._prepare_run(commit_every_rows) + effective_orchestrator = orchestrator or SyncPipelineOrchestrator() + with self._executing(prepared, commit_every_rows): + effective_orchestrator.run( + prepared.exec_dag, + observer=observer or NoOpObserver(), + run_id=prepared.run_id, + pipeline_uri=prepared.pipeline_uri, + materialize_results=materialize_results, + ) + return self._finish_run(prepared) + + async def run_async( + self, + orchestrator: "AsyncPipelineOrchestrator | None" = None, + observer: "ExecutionObserverProtocol | None" = None, + materialize_results: bool = True, + commit_every_rows: int = 100, + ) -> "PipelineJob": + """Execute the job with the async orchestrator, from async code. + + The awaitable form of ``run``: ``await job.run_async()`` runs on the caller's + event loop instead of blocking it. ``job.run(orchestrator=AsyncPipelineOrchestrator())`` + also works inside a running loop (a Jupyter kernel), but blocks that loop until + the run ends. Both store the same rows and record ids. Arguments, errors and + result are those of ``run``. + + Args: + orchestrator: The async orchestrator to run with. Defaults to + ``AsyncPipelineOrchestrator()``. + observer: Optional execution observer. + materialize_results: As for ``run``. + commit_every_rows: As for ``run``. + + Returns: + ``self``, as ``run`` returns. + + Raises: + TypeError: If ``orchestrator`` is not an ``AsyncPipelineOrchestrator``; + run a ``SyncPipelineOrchestrator`` with ``job.run``. + ValueError: As for ``run``. + RuntimeError: As for ``run``. + StoreWriteError: As for ``run``. + StoreReadError: As for ``run``. + """ + from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator + from orcapod.pipeline.observer import NoOpObserver + + if orchestrator is None: + orchestrator = AsyncPipelineOrchestrator() + elif not isinstance(orchestrator, AsyncPipelineOrchestrator): + raise TypeError( + "PipelineJob.run_async() runs an AsyncPipelineOrchestrator, got " + f"{type(orchestrator).__name__}. Use job.run(orchestrator=...) to run " + "it." + ) + prepared = self._prepare_run(commit_every_rows) + with self._executing(prepared, commit_every_rows): + await orchestrator.run_async( + prepared.exec_dag, + observer=observer or NoOpObserver(), + run_id=prepared.run_id, + pipeline_uri=prepared.pipeline_uri, + materialize_results=materialize_results, + ) + return self._finish_run(prepared) + + def _prepare_run(self, commit_every_rows: int) -> "_PreparedRun": + """Check the job can run and build what a run needs; shared by both run forms. + + Args: + commit_every_rows: As for ``run``. + + Returns: + The execution DAG, run id, pipeline URI and the store views of the run. + + Raises: + ValueError: As for ``run``. + RuntimeError: As for ``run``. + StoreError: If writes still pending from before the run cannot be committed. + """ + import hashlib + import uuid + if ( isinstance(commit_every_rows, bool) or not isinstance(commit_every_rows, int) @@ -955,8 +1061,6 @@ def run( self._persistent_node_map[v_hash], ) - # --- Execute --- - effective_observer = observer or NoOpObserver() run_id = uuid.uuid4().hex[:16] # Snapshot hash: SHA-256 of sink-node content digests for the run URI. @@ -988,28 +1092,56 @@ def run( flush_error.add_note(_PENDING_BEFORE_RUN_NOTE) raise flush_error from exc - effective_orchestrator = orchestrator or SyncPipelineOrchestrator() + return _PreparedRun( + exec_dag=exec_dag, + run_id=run_id, + pipeline_uri=pipeline_uri, + pipeline_db=pipeline_db, + result_db=result_db, + ) + + @contextlib.contextmanager + def _executing( + self, prepared: "_PreparedRun", commit_every_rows: int + ) -> Iterator[None]: + """Hold every function node to the run's commit cadence while the orchestrator runs. + + A failure inside commits the writes that succeeded before it, then propagates. + + Args: + prepared: The run, from ``_prepare_run``. + commit_every_rows: As for ``run``. + """ try: with contextlib.ExitStack() as batched: # Function nodes commit every commit_every_rows rows (ITL-919). - for node in exec_dag.nodes(): + for node in prepared.exec_dag.nodes(): if isinstance(node, FunctionJobNode): batched.enter_context( node._batched_commits(commit_every_rows) # pyright: ignore[reportPrivateUsage] # the job owns its nodes' commit cadence ) - effective_orchestrator.run( - exec_dag, - observer=effective_observer, - run_id=run_id, - pipeline_uri=pipeline_uri, - materialize_results=materialize_results, - ) + yield except Exception as run_error: # Commit the writes that did succeed before the failure, so their rows # do not look dropped, then report the failure itself. - flush_best_effort(run_error, pipeline_db, result_db) + flush_best_effort(run_error, prepared.pipeline_db, prepared.result_db) raise + def _finish_run(self, prepared: "_PreparedRun") -> "PipelineJob": + """Commit the run's last writes and mark the job as run. + + Args: + prepared: The run, from ``_prepare_run``. + + Returns: + ``self``. + + Raises: + StoreReadError: As for ``run``. + StoreWriteError: As for ``run``. + """ + pipeline_db = prepared.pipeline_db + result_db = prepared.result_db # Flush databases. Every view of the store shares one set of pending # batches, so a failed batch can leave the ones after it unwritten. pending_note = ( @@ -1031,7 +1163,7 @@ def run( # Mutate self in place. self._has_run = True - self._run_id = run_id + self._run_id = prepared.run_id return self diff --git a/superpowers/plans/2026-11-12-itl-994-997-job-run.md b/superpowers/plans/2026-11-12-itl-994-997-job-run.md index a428d68a2..c23b2526b 100644 --- a/superpowers/plans/2026-11-12-itl-994-997-job-run.md +++ b/superpowers/plans/2026-11-12-itl-994-997-job-run.md @@ -28,6 +28,20 @@ fragment per issue. (recommended), per-row sync log to DEBUG (recommended), names. - Allowed meanwhile: decision-independent failing tests, with the entry points and names kept in one place at the top of each test module. +- 2026-11-12, developer decisions (via coordinator): option (c); run/run_async return + RunSummary (breaking, no shim; changelog under "Breaking changes and migration" with the + one-line fix for `job = job.run()`); one RowFailuresWarning; sync per-row traceback to DEBUG; + names as proposed; docs in function-pods.md; export RunSummary and RowFailuresWarning and say + where in the PR body; manual ipykernel check; "run again" must read 0 computed, N from store. + Two commits (994 then 997), one push. Main merged at eeeb993e. +- ITL-994 done (commit "fix(pipeline): ...ITL-994"). Findings: Python 3.12's Thread.join marks + a running thread stopped when interrupted (bpo-45274 handling), so the bridge waits on an + Event; asyncio.run inside the `except RuntimeError` handler gave every run error a stray + __context__ (caught by test_store_write_failure), so it runs outside the handler. +- Manual ipykernel check (jupyter_client 8.6.3, ipykernel 6.30.1): sync, job.run(async) and + `await job.run_async()` give the same rows; interrupt after 1 s of 400 slow rows stops + job.run(async) with KeyboardInterrupt (252 rows called, none after) and run_async with + CancelledError (254, none after); the kernel stays usable. ## Findings (today, main 290889e8) @@ -58,12 +72,12 @@ fragment per issue. ## Task 1 (ITL-994): failing tests -- [ ] `test_job_run_in_event_loop.py`: a job run with the async orchestrator from inside a +- [x] `test_job_run_in_event_loop.py`: a job run with the async orchestrator from inside a running loop equals a run outside one (stored records minus per-run columns `__data_id`, `__pod_ts`; node table with all info), InMemory and Delta; `await job.run_async()` the same; a store error inside the bridged run reaches the caller with its type; KeyboardInterrupt while waiting cancels the run and re-raises. -- [ ] Run, see them fail with "asyncio.run() cannot be called from a running event loop". +- [x] Run, see them fail with "asyncio.run() cannot be called from a running event loop". ## Task 2 (ITL-994): implementation diff --git a/tests/test_pipeline/test_job_run_in_event_loop.py b/tests/test_pipeline/test_job_run_in_event_loop.py index 7f533e503..299da32f4 100644 --- a/tests/test_pipeline/test_job_run_in_event_loop.py +++ b/tests/test_pipeline/test_job_run_in_event_loop.py @@ -7,7 +7,9 @@ from __future__ import annotations +import _thread import asyncio +import time from collections.abc import Callable from pathlib import Path from typing import Any @@ -17,12 +19,13 @@ from orcapod import PipelineJob, function_pod from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase -from orcapod.pipeline import AsyncPipelineOrchestrator +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator from orcapod.sources import DictSource -# Columns that differ between any two runs by design: the datagram uuid and the time the -# result was computed. -PER_RUN_COLUMNS = ("__data_id", "__pod_ts") +# Columns that differ between any two runs by design, also between two script runs: the +# result's datagram uuid, the output's source-info token (which carries that uuid) and the +# time the result was computed. +PER_RUN_COLUMNS = ("__data_id", "_source_y", "__pod_ts") STORES: dict[str, Callable[[Path], Any]] = { "in_memory": lambda _: InMemoryArrowDatabase(), @@ -58,7 +61,7 @@ def make_job(store: Any) -> PipelineJob: return job -def stored_rows(job: PipelineJob) -> list[dict[str, Any]]: +def stored_rows(job: PipelineJob) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: """The node's stored records and its table, minus per-run columns, sorted by tag.""" records = job.nodes["doubled"].get_all_records(all_info=True) assert records is not None @@ -66,8 +69,11 @@ def stored_rows(job: PipelineJob) -> list[dict[str, Any]]: [c for c in PER_RUN_COLUMNS if c in records.column_names] ) table = job.nodes["doubled"].as_table(all_info=True) - joined = records.join(table, keys="k", join_type="inner", right_suffix="_table") - return sorted(joined.to_pylist(), key=lambda row: row["k"]) + + def by_tag(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + return sorted(rows, key=lambda row: row["k"]) + + return by_tag(records.to_pylist()), by_tag(table.to_pylist()) @pytest.mark.parametrize("store_kind", list(STORES)) @@ -134,3 +140,70 @@ async def two_cells() -> int: return pa.table({"n": [1]}).num_rows assert asyncio.run(two_cells()) == 1 + + +@pytest.mark.parametrize("store_kind", list(STORES)) +def test_await_run_async_stores_what_a_script_run_stores( + store_kind: str, tmp_path: Path +) -> None: + script_job = make_job(STORES[store_kind](tmp_path / "script")) + script_job.run(orchestrator=AsyncPipelineOrchestrator()) + + async_job = make_job(STORES[store_kind](tmp_path / "async")) + + async def async_code() -> None: + await async_job.run_async() + + asyncio.run(async_code()) + + assert stored_rows(async_job) == stored_rows(script_job) + + +def test_run_async_refuses_the_sync_orchestrator() -> None: + job = make_job(InMemoryArrowDatabase()) + with pytest.raises(TypeError, match=r"job\.run"): + _ = asyncio.run( + job.run_async( + orchestrator=SyncPipelineOrchestrator() # pyright: ignore[reportArgumentType] # the wrong orchestrator is the case under test + ) + ) + + +N_SLOW_ROWS = 200 # more than the default thread pool runs at once +slow_calls: list[int] = [] + + +@function_pod(output_keys="y") +def slow_and_interrupted(x: int) -> int: + """Press the interrupt button during the first row; take a while for every row.""" + slow_calls.append(x) + if len(slow_calls) == 1: + _thread.interrupt_main() + time.sleep(0.05) + return x + + +def test_interrupt_while_running_inside_a_loop_stops_the_run() -> None: + """The notebook's interrupt button stops the run; nothing keeps running after it.""" + slow_calls.clear() + rows = [{"k": f"r{i:02d}", "x": i} for i in range(N_SLOW_ROWS)] + job = PipelineJob(name="interrupt_demo", store=InMemoryArrowDatabase()) + with job: + source = DictSource(data=rows, tag_columns=["k"], source_id="rows") + slow_and_interrupted.pod(source, label="slow") + + async def notebook_cell() -> None: + job.run(orchestrator=AsyncPipelineOrchestrator()) + + # A bare loop, as a kernel has: asyncio.run would install its own SIGINT handler. + loop = asyncio.new_event_loop() + try: + with pytest.raises(KeyboardInterrupt): + loop.run_until_complete(notebook_cell()) + finally: + loop.close() + + calls_when_stopped = len(slow_calls) + assert calls_when_stopped < N_SLOW_ROWS + time.sleep(0.2) + assert len(slow_calls) == calls_when_stopped From 66bbcf035ebf8ba127411d807848f2bd33184fa8 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 16:41:59 -0800 Subject: [PATCH 3/6] feat(pipeline)!: job.run returns a run summary and warns when rows failed (ITL-997) A function that raised for every row left its node empty while job.run() returned normally: the sync orchestrator logged one traceback per row and the async one nothing visible. PipelineJob.run and run_async now return a RunSummary (per function node: rows computed, read from the store and failed, and the first error), shown as a table in a notebook and kept as job.last_run; print_traceback prints a node's first failed row's traceback. A run with failed rows issues one RowFailuresWarning; a clean run is quiet. Rows are counted by an observer composed with the user's one, so nodes and orchestrators are unchanged; the sync per-row log moves to DEBUG. BREAKING CHANGE: run() returns the summary instead of the job. Write job.run() where code had job = job.run(), and read results from the job. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-997.md | 27 ++ docs/concepts/function-pods.md | 62 ++++ src/orcapod/core/nodes/function_node.py | 6 +- src/orcapod/errors.py | 11 + src/orcapod/pipeline/__init__.py | 3 + src/orcapod/pipeline/job.py | 100 +++++-- src/orcapod/pipeline/run_summary.py | 283 ++++++++++++++++++ .../plans/2026-11-12-itl-994-997-job-run.md | 5 + .../test_pipeline_async_integration.py | 16 +- .../test_function_node_nullability.py | 12 +- tests/test_pipeline/test_aggregation_job.py | 6 +- tests/test_pipeline/test_drop_nulls_job.py | 3 +- tests/test_pipeline/test_integration_smoke.py | 8 +- tests/test_pipeline/test_job_run_summary.py | 53 +++- tests/test_pipeline/test_pipeline.py | 56 ++-- tests/test_pipeline/test_pipeline_job.py | 82 ++--- tests/test_pipeline/test_serialization.py | 8 +- .../test_pipeline/test_store_write_failure.py | 6 +- 18 files changed, 626 insertions(+), 121 deletions(-) create mode 100644 changelog.d/ITL-997.md create mode 100644 src/orcapod/pipeline/run_summary.py diff --git a/changelog.d/ITL-997.md b/changelog.d/ITL-997.md new file mode 100644 index 000000000..b6b2c047f --- /dev/null +++ b/changelog.d/ITL-997.md @@ -0,0 +1,27 @@ +### Breaking changes and migration + +- **`PipelineJob.run()` returns a `RunSummary`, not the job.** Results are read from the + job itself, as before (`job.nodes[...]`). Code written `job = job.run()` becomes + `job.run()`; code that kept the return value as the job (`completed = job.run()`, + `completed.nodes[...]`) uses the job instead. `run_async()` returns the same summary. + (ITL-997) + +### Added + +- **`RunSummary`** (`orcapod.pipeline`) — what a job run did, per function node: rows + computed, rows read from the store, rows failed, and the first error (`NodeRunSummary`). + Shown as a table at the end of a notebook cell and as text by `print`; kept as + `job.last_run`. `job.last_run.print_traceback("")` prints the full traceback of a + node's first failed row. (ITL-997) +- **`RowFailuresWarning`** (`orcapod.errors`) — issued once at the end of a job run in which + rows failed (with the default `error_policy="continue"`), naming each node with failed + rows, how many failed out of how many, the first error, and how to see its traceback. A + run with no failed rows issues none. (ITL-997) + +### Changed + +- **A failed row no longer logs its traceback at WARNING on the sync orchestrator.** The + per-row log is at DEBUG, as on the async orchestrator, which logged nothing visible: a + function failing on every row printed one traceback per row on the sync path and nothing + at all on the async path. The run summary and `RowFailuresWarning` now report failures the + same way on both. `error_policy="fail_fast"` is unchanged. (ITL-997) diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index dfa5f4c8a..a86194448 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -451,6 +451,68 @@ print(job.nodes["bmi"].as_table().num_rows) # 3 ``` +### Reading the run summary + +`job.run()` (and `await job.run_async()`) returns a `RunSummary`: for each function node, how +many rows were computed, how many were read from the store, and how many failed. At the end +of a notebook cell it is shown as a table; `print` shows the same as text. The job keeps it as +`job.last_run`. The results themselves are read from `job.nodes`, as above. + +Running the same job again computes nothing new, and the summary says so: + +```python +job = PipelineJob(name="bmi_demo", store=db) +with job: + source = DictSource(data=mice, tag_columns=["subject_id"], source_id="mice") + compute_bmi.pod(source, label="bmi") + +summary = job.run() +print(summary) +# Run of job 'bmi_demo': all rows succeeded. +# bmi: 3 rows: 0 computed, 3 from store, 0 failed +``` + +When a function raises for a row, the run skips that row and goes on with the others (the +default `error_policy="continue"`). At the end it issues one `RowFailuresWarning` (from +`orcapod.errors`) naming each node with failed rows, and the summary shows the first error. +A run with no failed rows issues no warning: + +```python +@function_pod(output_keys="bmi") +def careless_bmi(weight: float, height: float) -> float: + return weight / heigth ** 2 # a typo: heigth + + +job = PipelineJob(name="bmi_typo", store=InMemoryArrowDatabase()) +with job: + source = DictSource(data=mice, tag_columns=["subject_id"], source_id="mice") + careless_bmi.pod(source, label="bmi") + +summary = job.run() +# RowFailuresWarning: 3 of 3 rows failed in node 'bmi' (first error: NameError: name 'heigth' is not defined). Full traceback: job.last_run.print_traceback("bmi"). +print(summary) +# Run of job 'bmi_typo': 3 rows failed. +# bmi: 3 rows: 0 computed, 0 from store, 3 failed +# first error: NameError: name 'heigth' is not defined +# Full traceback: job.last_run.print_traceback("bmi") +``` + +`job.last_run.print_traceback("bmi")` prints the full traceback of the node's first failed +row, ending in the line of your function that raised. Fix the function and run the job +again: the rows that succeeded before are read from the store, and only the failed ones +are computed. To stop a run at the first failed row instead, pass +`SyncPipelineOrchestrator(error_policy="fail_fast")` as the orchestrator. + +The counts are also available one by one, for example in a test: + +```python +node = summary.nodes["bmi"] +print(node.computed, node.reused, node.failed, type(node.first_error).__name__) +# 0 0 3 NameError +print(summary.ok, summary.failed_rows) +# False 3 +``` + ### When results are committed Inside `job.run()`, each function node commits its results and pipeline records to the diff --git a/src/orcapod/core/nodes/function_node.py b/src/orcapod/core/nodes/function_node.py index 888d0b0df..0882ee544 100644 --- a/src/orcapod/core/nodes/function_node.py +++ b/src/orcapod/core/nodes/function_node.py @@ -1760,8 +1760,10 @@ def execute( ctx_obs.on_node_end(node_label, node_hash) raise except Exception as exc: - logger.warning( - "Data execution failed in %s: %s", + # Reported at the end of a job run (RunSummary, RowFailuresWarning), + # so one traceback per failed row is only for debugging. + logger.debug( + "Data execution failed in %s, skipping: %s", node_label, exc, exc_info=True, diff --git a/src/orcapod/errors.py b/src/orcapod/errors.py index d6150dd15..1923bbffc 100644 --- a/src/orcapod/errors.py +++ b/src/orcapod/errors.py @@ -330,6 +330,17 @@ class PipelineJobRequiredError(RuntimeError): """ +class RowFailuresWarning(UserWarning): + """Issued once at the end of a job run in which rows failed. + + With the default ``error_policy="continue"``, a row whose function raises is skipped + and the run goes on. The warning names each node with failed rows, how many failed out + of how many, and the first error, and says how to print its full traceback. The run's + ``RunSummary`` (returned by ``PipelineJob.run``, kept as ``job.last_run``) holds the + same counts. + """ + + class EmptyDataAccessError(Exception): """Raised when a payload-access method is called on an ``EmptyData`` instance. diff --git a/src/orcapod/pipeline/__init__.py b/src/orcapod/pipeline/__init__.py index 8f4a05551..c539aa0a2 100644 --- a/src/orcapod/pipeline/__init__.py +++ b/src/orcapod/pipeline/__init__.py @@ -5,6 +5,7 @@ from .graph import Pipeline from .job import PipelineJob from .logging_observer import LoggingObserver, DataLogger +from .run_summary import NodeRunSummary, RunSummary from .serialization import ( LoadStatus, PIPELINE_FORMAT_VERSION, @@ -20,12 +21,14 @@ "GraphProtocol", "LoadStatus", "LoggingObserver", + "NodeRunSummary", "DataLogger", "OrcaDAG", "PIPELINE_FORMAT_VERSION", "PIPELINE_JOB_FORMAT_VERSION", "Pipeline", "PipelineJob", + "RunSummary", "StatusObserver", "SyncPipelineOrchestrator", ] diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index 906ad1283..f7b26e2db 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -4,6 +4,7 @@ import contextlib import logging +import warnings from collections.abc import Iterator from dataclasses import dataclass from typing import TYPE_CHECKING, Any @@ -19,6 +20,7 @@ unclaimed_tables_written_before_v03, ) from orcapod.errors import ( + RowFailuresWarning, StoreError, StoreReadError, StoreWriteError, @@ -26,6 +28,15 @@ ) from orcapod.pipeline.base import AbstractPipelineBase from orcapod.pipeline.dag import OrcaDAG +from orcapod.pipeline.run_summary import ( + NodeRunSummary, + RunCountingObserver, + RunSummary, +) +from orcapod.protocols.node_protocols import ( + is_function_node, + is_side_effect_function_node, +) from orcapod.protocols import core_protocols as cp from orcapod.databases.utils import as_table_committing from orcapod.types import CacheMode, NodeConfig @@ -69,6 +80,25 @@ class _PreparedRun: result_db: "ArrowDatabaseProtocol" +def _with_counter( + observer: "ExecutionObserverProtocol | None", counter: RunCountingObserver +) -> "ExecutionObserverProtocol": + """The observer a run passes to its orchestrator: the user's, plus the row counter. + + Args: + observer: The observer the user passed to ``run``, if any. + counter: The run's row counter. + + Returns: + ``counter`` alone, or a ``CompositeObserver`` calling ``observer`` then ``counter``. + """ + if observer is None: + return counter + from orcapod.pipeline.composite_observer import CompositeObserver + + return CompositeObserver(observer, counter) # pyright: ignore[reportReturnType] # forwards every hook; its **kwargs signatures do not match the protocol structurally + + class PipelineJob(AbstractPipelineBase[JobNode]): """Pipeline + source bindings + execution context. @@ -126,6 +156,8 @@ def __init__( self._sources_by_hash: dict[str, cp.StreamProtocol] = {} self._has_run: bool = False self._run_id: str | None = None + # The summary of the last run this object completed (ITL-997). + self._last_run: RunSummary | None = None # True for a job built by load(): it reads stored results and never runs. self._loaded: bool = False @@ -198,6 +230,17 @@ def compile(self) -> None: # compiled_pipeline — lazy cache # ------------------------------------------------------------------ + @property + def last_run(self) -> RunSummary | None: + """The summary of the last run this job completed, or ``None`` before one. + + The same object ``run`` and ``run_async`` returned: per function node, the rows + computed, read from the store and failed, and each node's first error. Use + ``job.last_run.print_traceback("")`` to see a failed row's full + traceback. A run that raised leaves it unchanged. A loaded job has none. + """ + return self._last_run + @property def compiled_pipeline(self) -> "Pipeline": """The lightweight ``Pipeline`` blueprint for this job (lazy cache). @@ -794,7 +837,7 @@ def run( observer: "ExecutionObserverProtocol | None" = None, materialize_results: bool = True, commit_every_rows: int = 100, - ) -> "PipelineJob": + ) -> RunSummary: """Execute the resolvable subgraph of this job in place. Nodes whose upstream includes an unbound source (and all their @@ -843,9 +886,16 @@ def run( once one table's pending rows reach that size, which would break the node's results-before-records commit order. + Rows whose function raised (with the default ``error_policy="continue"``) are + skipped, and the run goes on. At the end, if any row failed, a single + ``RowFailuresWarning`` names each node with failed rows, how many failed, the + first error, and how to print its traceback. A run without failures is quiet. + Returns: - ``self`` — the same ``PipelineJob`` instance, with - ``_has_run`` and ``_run_id`` set. + The run's ``RunSummary``: per function node, the rows computed, read from + the store and failed, with each node's first error. Shown at the end of a + notebook cell; also kept as ``job.last_run``. Results are read from + ``job.nodes``. Raises: ValueError: If no store is set, or ``commit_every_rows`` is not a @@ -865,20 +915,23 @@ def run( was needed to write a node's results. Its rows are not recomputed and the table is not overwritten. """ - from orcapod.pipeline.observer import NoOpObserver from orcapod.pipeline.sync_orchestrator import SyncPipelineOrchestrator prepared = self._prepare_run(commit_every_rows) + counter = RunCountingObserver() effective_orchestrator = orchestrator or SyncPipelineOrchestrator() with self._executing(prepared, commit_every_rows): - effective_orchestrator.run( + _ = effective_orchestrator.run( prepared.exec_dag, - observer=observer or NoOpObserver(), + observer=_with_counter(observer, counter), run_id=prepared.run_id, pipeline_uri=prepared.pipeline_uri, materialize_results=materialize_results, ) - return self._finish_run(prepared) + summary = self._finish_run(prepared, counter) + if not summary.ok: + warnings.warn(summary.failures_message(), RowFailuresWarning, stacklevel=2) + return summary async def run_async( self, @@ -886,7 +939,7 @@ async def run_async( observer: "ExecutionObserverProtocol | None" = None, materialize_results: bool = True, commit_every_rows: int = 100, - ) -> "PipelineJob": + ) -> RunSummary: """Execute the job with the async orchestrator, from async code. The awaitable form of ``run``: ``await job.run_async()`` runs on the caller's @@ -903,7 +956,7 @@ async def run_async( commit_every_rows: As for ``run``. Returns: - ``self``, as ``run`` returns. + The run's ``RunSummary``, as ``run`` returns. Raises: TypeError: If ``orchestrator`` is not an ``AsyncPipelineOrchestrator``; @@ -914,7 +967,6 @@ async def run_async( StoreReadError: As for ``run``. """ from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator - from orcapod.pipeline.observer import NoOpObserver if orchestrator is None: orchestrator = AsyncPipelineOrchestrator() @@ -925,15 +977,19 @@ async def run_async( "it." ) prepared = self._prepare_run(commit_every_rows) + counter = RunCountingObserver() with self._executing(prepared, commit_every_rows): - await orchestrator.run_async( + _ = await orchestrator.run_async( prepared.exec_dag, - observer=observer or NoOpObserver(), + observer=_with_counter(observer, counter), run_id=prepared.run_id, pipeline_uri=prepared.pipeline_uri, materialize_results=materialize_results, ) - return self._finish_run(prepared) + summary = self._finish_run(prepared, counter) + if not summary.ok: + warnings.warn(summary.failures_message(), RowFailuresWarning, stacklevel=2) + return summary def _prepare_run(self, commit_every_rows: int) -> "_PreparedRun": """Check the job can run and build what a run needs; shared by both run forms. @@ -1127,14 +1183,17 @@ def _executing( flush_best_effort(run_error, prepared.pipeline_db, prepared.result_db) raise - def _finish_run(self, prepared: "_PreparedRun") -> "PipelineJob": - """Commit the run's last writes and mark the job as run. + def _finish_run( + self, prepared: "_PreparedRun", counter: RunCountingObserver + ) -> RunSummary: + """Commit the run's last writes, mark the job as run and keep its summary. Args: prepared: The run, from ``_prepare_run``. + counter: The observer that counted the run's rows. Returns: - ``self``. + The run's summary, also kept as ``last_run``. Raises: StoreReadError: As for ``run``. @@ -1164,8 +1223,13 @@ def _finish_run(self, prepared: "_PreparedRun") -> "PipelineJob": # Mutate self in place. self._has_run = True self._run_id = prepared.run_id - - return self + node_summaries: list[NodeRunSummary] = [] + for node in prepared.exec_dag.topological_sort(): + label: str = node.label + if is_function_node(node) or is_side_effect_function_node(node): + node_summaries.append(counter.node_summary(label)) + self._last_run = RunSummary("/".join(self._name), node_summaries) + return self._last_run def __repr__(self) -> str: n_sources = len(self._sources) diff --git a/src/orcapod/pipeline/run_summary.py b/src/orcapod/pipeline/run_summary.py new file mode 100644 index 000000000..1967d64e7 --- /dev/null +++ b/src/orcapod/pipeline/run_summary.py @@ -0,0 +1,283 @@ +"""What a job run did with each row: the summary ``PipelineJob.run`` returns. + +With the default ``error_policy="continue"``, a row whose function raises is skipped and +the run goes on. ``RunSummary`` tells the user, per function node, how many rows were +computed, how many were read from the store, and how many failed with which first error. +""" + +from __future__ import annotations + +import html +import threading +import traceback +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from orcapod.core.result_cache import ResultCache +from orcapod.pipeline.observer import NoOpLogger + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + + from orcapod.protocols.core_protocols import DataProtocol, TagProtocol + from orcapod.types import SchemaLike + +_NOOP_LOGGER = NoOpLogger() + + +@dataclass(frozen=True) +class NodeRunSummary: + """What one function node did with its rows in a run. + + Attributes: + label: The node's label, its key in ``job.nodes``. + computed: Rows whose function ran and returned. + reused: Rows whose result was read from the store instead. + failed: Rows whose function raised; the run skipped them. + first_error: The exception of the first failed row, with its traceback, or + ``None`` when no row failed. + """ + + label: str + computed: int + reused: int + failed: int + first_error: BaseException | None + + @property + def rows(self) -> int: + """All rows the node received: computed, reused and failed.""" + return self.computed + self.reused + self.failed + + +class RunSummary: + """What a job run did, per function node; returned by ``PipelineJob.run``. + + Shown at the end of a notebook cell, it lists every function node with its counts and, + for a node with failed rows, the first error and how to see its full traceback. The job + also keeps it as ``job.last_run``. + + Attributes: + job_name: The job's name. + nodes: Per function node, in run order, keyed by label. + """ + + def __init__(self, job_name: str, nodes: Sequence[NodeRunSummary]) -> None: + """Build the summary. + + Args: + job_name: The job's name. + nodes: One entry per function node of the run, in run order. + """ + self.job_name: str = job_name + self.nodes: Mapping[str, NodeRunSummary] = {node.label: node for node in nodes} + + @property + def failed_rows(self) -> int: + """The number of rows that failed, over every node.""" + return sum(node.failed for node in self.nodes.values()) + + @property + def ok(self) -> bool: + """``True`` when no row failed.""" + return self.failed_rows == 0 + + def failures_message(self) -> str: + """One sentence per node with failed rows, for the run's warning. + + Returns: + The sentences joined, or ``""`` when no row failed. + """ + return " ".join( + f"{node.failed} of {_rows(node.rows)} failed in node {node.label!r} " + f"(first error: {_error_line(node.first_error)}). " + f"Full traceback: {_traceback_call(node.label)}." + for node in self.nodes.values() + if node.failed + ) + + def print_traceback(self, label: str | None = None) -> None: + """Print the full traceback of a node's first failed row to standard error. + + Args: + label: The node; by default the first node with a failed row. + + Raises: + KeyError: If no function node of the run has that label. + """ + if label is not None and label not in self.nodes: + raise KeyError( + f"No function node {label!r} in this run; its function nodes are " + f"{sorted(self.nodes)}." + ) + candidates = [self.nodes[label]] if label is not None else self.nodes.values() + for node in candidates: + if node.first_error is not None: + traceback.print_exception(node.first_error) + return + where = f" in node {label!r}" if label is not None else "" + print(f"No rows failed{where} in this run.") + + def _headline(self) -> str: + if self.failed_rows: + outcome = f"{_rows(self.failed_rows)} failed." + else: + outcome = "all rows succeeded." + return f"Run of job {self.job_name!r}: {outcome}" + + def __repr__(self) -> str: + lines = [self._headline()] + for node in self.nodes.values(): + lines.append( + f" {node.label}: {_rows(node.rows)}: {node.computed} computed, " + f"{node.reused} from store, {node.failed} failed" + ) + if node.failed: + lines.append(f" first error: {_error_line(node.first_error)}") + lines.append(f" Full traceback: {_traceback_call(node.label)}") + return "\n".join(lines) + + def _repr_html_(self) -> str: + """The notebook display: the same content as ``repr``, as a table.""" + header = "".join( + f"{name}" + for name in ( + "node", + "rows", + "computed", + "from store", + "failed", + "first error", + ) + ) + body = [] + for node in self.nodes.values(): + failed_style = ( + " style='color:#c62828;font-weight:bold'" if node.failed else "" + ) + error = "" + if node.failed: + error = ( + f"{html.escape(_error_line(node.first_error))}
" + f"{html.escape(_traceback_call(node.label))}" + ) + body.append( + "" + f"{html.escape(node.label)}" + f"{node.rows}{node.computed}{node.reused}" + f"{node.failed}{error}" + "" + ) + return ( + f"

{html.escape(self._headline())}

" + f"{header}" + f"{''.join(body)}
" + ) + + +def _rows(n: int) -> str: + return f"{n} row" if n == 1 else f"{n} rows" + + +def _error_line(error: BaseException | None) -> str: + """``Type: first line of the message``.""" + if error is None: + return "" + message = str(error) + lines = message.splitlines() + first = lines[0] if lines else "" + if len(lines) > 1: + first += " ..." + return f"{type(error).__name__}: {first}" if first else type(error).__name__ + + +def _traceback_call(label: str) -> str: + return f'job.last_run.print_traceback("{label}")' + + +class RunCountingObserver: + """Counts, per node label, the rows computed, reused and failed in one run. + + ``PipelineJob.run`` runs it beside the user's observer. Every contextualized copy is + this observer itself, so all nodes count into one place. Hooks may be called from + several threads, so the counts are updated under a lock. + """ + + def __init__(self) -> None: + """Start with no counts.""" + self._lock: threading.Lock = threading.Lock() + self._computed: dict[str, int] = {} + self._reused: dict[str, int] = {} + self._failed: dict[str, int] = {} + self._first_error: dict[str, BaseException] = {} + + def node_summary(self, label: str) -> NodeRunSummary: + """The counts of node ``label``; zeros when it saw no rows. + + Args: + label: The node's label. + + Returns: + The node's summary. + """ + with self._lock: + return NodeRunSummary( + label=label, + computed=self._computed.get(label, 0), + reused=self._reused.get(label, 0), + failed=self._failed.get(label, 0), + first_error=self._first_error.get(label), + ) + + def contextualize(self, *identity_path: str) -> RunCountingObserver: + """Return this observer: counts are keyed by node label, not identity path.""" + return self + + def on_run_start(self, run_id: str, pipeline_uri: str = "") -> None: + """Nothing to count.""" + + def on_run_end(self, run_id: str) -> None: + """Nothing to count.""" + + def on_node_start( + self, node_label: str, node_hash: str, tag_schema: SchemaLike | None = None + ) -> None: + """Nothing to count.""" + + def on_node_end(self, node_label: str, node_hash: str) -> None: + """Nothing to count.""" + + def on_data_start( + self, node_label: str, tag: TagProtocol, data: DataProtocol + ) -> None: + """Nothing to count until the row ends.""" + + def on_data_end( + self, + node_label: str, + tag: TagProtocol, + input_data: DataProtocol, + output_data: DataProtocol | None, + cached: bool, + ) -> None: + """Count the row as reused (from the pipeline or result store) or computed.""" + from_store = cached or ( + output_data is not None + and output_data.get_meta_value(ResultCache.RESULT_COMPUTED_FLAG, True) + is False + ) + counts = self._reused if from_store else self._computed + with self._lock: + counts[node_label] = counts.get(node_label, 0) + 1 + + def on_data_crash( + self, node_label: str, tag: TagProtocol, data: DataProtocol, error: Exception + ) -> None: + """Count the row as failed; keep the node's first error.""" + with self._lock: + self._failed[node_label] = self._failed.get(node_label, 0) + 1 + _ = self._first_error.setdefault(node_label, error) + + def create_data_logger(self, tag: TagProtocol, data: DataProtocol) -> Any: + """No logging of its own.""" + return _NOOP_LOGGER diff --git a/superpowers/plans/2026-11-12-itl-994-997-job-run.md b/superpowers/plans/2026-11-12-itl-994-997-job-run.md index c23b2526b..0e31e2c14 100644 --- a/superpowers/plans/2026-11-12-itl-994-997-job-run.md +++ b/superpowers/plans/2026-11-12-itl-994-997-job-run.md @@ -42,6 +42,11 @@ fragment per issue. `await job.run_async()` give the same rows; interrupt after 1 s of 400 slow rows stops job.run(async) with KeyboardInterrupt (252 rows called, none after) and run_async with CancelledError (254, none after); the kernel stays usable. +- ITL-997 done (commit "feat(pipeline)!: ...ITL-997"): RunSummary/NodeRunSummary in + pipeline/run_summary.py (exported from orcapod.pipeline), RowFailuresWarning in + orcapod.errors, job.last_run, RunCountingObserver composed with the user's observer, sync + per-row log at DEBUG; 38 test sites updated; empty-input node reports zeros (coordinator ask). + Full suite before merge: 10107 passed, 116 skipped, 23 xfailed. ## Findings (today, main 290889e8) diff --git a/tests/test_channels/test_pipeline_async_integration.py b/tests/test_channels/test_pipeline_async_integration.py index e1a8186bc..71dcf2187 100644 --- a/tests/test_channels/test_pipeline_async_integration.py +++ b/tests/test_channels/test_pipeline_async_integration.py @@ -151,9 +151,9 @@ async def test_orchestrator_run_async_from_event_loop(self): def test_sync_run_then_db_retrieval(self): """Baseline: sync run() populates the DB for later retrieval.""" job = _build_job() - result_job = job.run() + _ = job.run() - records = result_job.nodes["letter_grade"].get_all_records() + records = job.nodes["letter_grade"].get_all_records() assert records is not None assert records.num_rows == 5 assert _grades_from_table(records) == EXPECTED @@ -162,8 +162,8 @@ def test_sync_and_async_produce_identical_results(self): """Sync and async execution paths yield the same grades.""" # Sync path sync_job = _build_job() - sync_result = sync_job.run() - sync_records = sync_result.nodes["letter_grade"].get_all_records() + _ = sync_job.run() + sync_records = sync_job.nodes["letter_grade"].get_all_records() assert sync_records is not None sync_grades = _grades_from_table(sync_records) @@ -199,8 +199,8 @@ def _system_tag_data(self, table: pa.Table) -> dict[str, list]: def test_join_pipeline_system_tags_identical(self): """Join pipeline: sync and async produce the same system-tag columns.""" sync_job = _build_job() - sync_result = sync_job.run() - sync_records = sync_result.nodes["letter_grade"].get_all_records( + _ = sync_job.run() + sync_records = sync_job.nodes["letter_grade"].get_all_records( columns={"system_tags": True} ) assert sync_records is not None @@ -254,8 +254,8 @@ def test_all_system_tag_columns_match_between_sync_and_async(self): from this comparison. """ sync_job = _build_job() - sync_result = sync_job.run() - sync_records = sync_result.nodes["letter_grade"].get_all_records( + _ = sync_job.run() + sync_records = sync_job.nodes["letter_grade"].get_all_records( columns={"system_tags": True} ) assert sync_records is not None diff --git a/tests/test_data/test_polars_nullability/test_function_node_nullability.py b/tests/test_data/test_polars_nullability/test_function_node_nullability.py index 428581e0f..53f56e27b 100644 --- a/tests/test_data/test_polars_nullability/test_function_node_nullability.py +++ b/tests/test_data/test_polars_nullability/test_function_node_nullability.py @@ -49,9 +49,9 @@ def double(x: int) -> int: with job: double.pod(source) - result_job = job.run() + _ = job.run() - fn_nodes = _get_function_nodes(result_job) + fn_nodes = _get_function_nodes(job) assert len(fn_nodes) == 1, "Expected exactly one FunctionNode" fn_node = fn_nodes[0] @@ -84,9 +84,9 @@ def triple(x: int) -> int: with job: triple.pod(source) - result_job = job.run() + _ = job.run() - fn_nodes = _get_function_nodes(result_job) + fn_nodes = _get_function_nodes(job) fn_node = fn_nodes[0] table = fn_node.get_all_records() @@ -126,9 +126,9 @@ def add_one(x: int) -> int: with job: add_one.pod(source) - result_job = job.run() + _ = job.run() - fn_nodes = _get_function_nodes(result_job) + fn_nodes = _get_function_nodes(job) fn_node = fn_nodes[0] # Force a DB-backed iteration by going through _load_cached_entries diff --git a/tests/test_pipeline/test_aggregation_job.py b/tests/test_pipeline/test_aggregation_job.py index 8db345840..a802ed501 100644 --- a/tests/test_pipeline/test_aggregation_job.py +++ b/tests/test_pipeline/test_aggregation_job.py @@ -96,7 +96,8 @@ def _run(store, source, operator, name, function=count_paths): job = PipelineJob(name=name, store=store) with job: pod(operator(source, label="agg"), label="counter") - return job.run() + _ = job.run() + return job class TestGroupByInJob: @@ -260,7 +261,8 @@ def _run(store, source, group_by): grouped = group_by(source, label="agg") selected = SelectDataColumns(columns=["path"])(grouped, label="sel") pod(selected, label="counter") - return job.run() + _ = job.run() + return job def test_legacy_fixture_has_the_old_shape(self, session_source_factory): """Guard the fixture -- if this stops holding, the next test is vacuous.""" diff --git a/tests/test_pipeline/test_drop_nulls_job.py b/tests/test_pipeline/test_drop_nulls_job.py index 47bdf8dc6..b0ab4ba3f 100644 --- a/tests/test_pipeline/test_drop_nulls_job.py +++ b/tests/test_pipeline/test_drop_nulls_job.py @@ -70,7 +70,8 @@ def _run(store: DeltaTableDatabase, timestamps: list[int | None]) -> PipelineJob "recording_start_timestamp", label="present" ) pod(present, label="seconds") - return job.run() + _ = job.run() + return job def _seconds(job: PipelineJob) -> dict[str, float]: diff --git a/tests/test_pipeline/test_integration_smoke.py b/tests/test_pipeline/test_integration_smoke.py index 8127cdce3..2ba1a8ba6 100644 --- a/tests/test_pipeline/test_integration_smoke.py +++ b/tests/test_pipeline/test_integration_smoke.py @@ -40,8 +40,8 @@ def test_smoke_run_completes(): with job: pod(_make_source(), label="doubler") - completed = job.run() - assert completed._has_run is True + _ = job.run() + assert job._has_run is True def test_smoke_nodes_accessible_after_run(): @@ -54,8 +54,8 @@ def test_smoke_nodes_accessible_after_run(): with job: pod(_make_source(), label="adder") - completed = job.run() - assert "adder" in completed.pipeline.nodes + _ = job.run() + assert "adder" in job.pipeline.nodes def test_smoke_run_raises_without_pipeline(): diff --git a/tests/test_pipeline/test_job_run_summary.py b/tests/test_pipeline/test_job_run_summary.py index 9843763aa..48062cc68 100644 --- a/tests/test_pipeline/test_job_run_summary.py +++ b/tests/test_pipeline/test_job_run_summary.py @@ -18,31 +18,37 @@ from orcapod import PipelineJob, function_pod from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase from orcapod.errors import RowFailuresWarning -from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.pipeline import ( + AsyncPipelineOrchestrator, + RunSummary, + SyncPipelineOrchestrator, +) from orcapod.sources import DictSource # --- The public names this feature adds, in one place --------------------------------- -def run(job: PipelineJob, orchestrator: Any) -> Any: +def run(job: PipelineJob, orchestrator: Any) -> RunSummary: """Run ``job``; return what the user gets back (the run summary).""" return job.run(orchestrator=orchestrator) -def node_counts(summary: Any, label: str) -> tuple[int, int, int]: +def node_counts(summary: RunSummary, label: str) -> tuple[int, int, int]: """``(computed, from store, failed)`` rows of node ``label`` in ``summary``.""" node = summary.nodes[label] return node.computed, node.reused, node.failed -def first_error(summary: Any, label: str) -> BaseException | None: +def first_error(summary: RunSummary, label: str) -> BaseException | None: """The first error raised in node ``label`` during the run.""" return summary.nodes[label].first_error -def last_run(job: PipelineJob) -> Any: +def last_run(job: PipelineJob) -> RunSummary: """The summary of the job's last completed run.""" - return job.last_run + summary = job.last_run + assert summary is not None + return summary # -------------------------------------------------------------------------------------- @@ -182,3 +188,38 @@ def test_notebook_representation_names_the_failing_node() -> None: assert "evens" in html assert "ValueError" in html assert "odd x=" in html + + +def test_run_async_returns_the_same_summary(store: Any) -> None: + import asyncio + + job = make_job(store, even_only, "evens") + with pytest.warns(RowFailuresWarning, match="2 of 4 rows failed in node 'evens'"): + summary = asyncio.run(job.run_async()) + + assert node_counts(summary, "evens") == (2, 0, 2) + assert last_run(job) is summary + + +@function_pod(output_keys="z") +def plus_one(y: int) -> int: + """Return ``y + 1``.""" + return y + 1 + + +def test_node_with_empty_input_counts_zero_rows(store: Any, orchestrator: Any) -> None: + """Every row fails upstream, so the next node receives none: it reports zeros.""" + rows = [{"k": f"r{i}", "x": i} for i in range(4)] + job = PipelineJob(name="summary_chain", store=store) + with job: + source = DictSource(data=rows, tag_columns=["k"], source_id="rows") + broken = always_fails.pod(source, label="broken") + plus_one.pod(broken, label="after") + + with pytest.warns(RowFailuresWarning, match="4 of 4 rows failed in node 'broken'"): + summary = run(job, orchestrator) + + assert list(summary.nodes) == ["broken", "after"] + assert node_counts(summary, "broken") == (0, 0, 4) + assert node_counts(summary, "after") == (0, 0, 0) + assert "after: 0 rows: 0 computed, 0 from store, 0 failed" in repr(summary) diff --git a/tests/test_pipeline/test_pipeline.py b/tests/test_pipeline/test_pipeline.py index 8f073772b..0593b14d5 100644 --- a/tests/test_pipeline/test_pipeline.py +++ b/tests/test_pipeline/test_pipeline.py @@ -230,10 +230,10 @@ def test_exec_nodes_have_pipeline_database_after_run(self, pipeline_db): with job: joined = Join()(src_a, src_b) pod(joined, label="adder") - result = job.run() + _ = job.run() - # After run, nodes["adder"] is the exec node in the returned result - exec_node = result.nodes["adder"] + # After run, nodes["adder"] is the exec node + exec_node = job.nodes["adder"] assert isinstance(exec_node, FunctionJobNode) assert exec_node._pipeline_database is not None @@ -243,9 +243,9 @@ def test_exec_operator_nodes_have_pipeline_database_after_run(self, pipeline_db) job = PipelineJob(store=pipeline_db) with job: Join()(src_a, src_b, label="joiner") - result = job.run() + _ = job.run() - exec_node = result.nodes["joiner"] + exec_node = job.nodes["joiner"] assert isinstance(exec_node, OperatorJobNode) assert exec_node._pipeline_database is not None @@ -265,9 +265,9 @@ def test_result_database_scoped_to_pipeline_name(self, pipeline_db): with job: joined = Join()(src_a, src_b) pod(joined, label="adder") - result = job.run() + _ = job.run() - exec_node = result.nodes["adder"] + exec_node = job.nodes["adder"] assert isinstance(exec_node, FunctionJobNode) # Verify the exec node has databases attached assert exec_node._pipeline_database is not None @@ -368,10 +368,10 @@ def test_run_executes_all_nodes(self, pipeline_db): joined = Join()(src_a, src_b) pod(joined, label="adder") - result = job.run() + _ = job.run() - # After run, function node should have records (in the returned result job) - node = result.nodes["adder"] + # After run, function node should have records + node = job.nodes["adder"] records = node.get_all_records() assert records is not None assert records.num_rows == 2 # two input rows (a, b) @@ -387,14 +387,14 @@ def test_pipeline_path_prefix_scoping(self, pipeline_db): joined = Join()(src_a, src_b, label="joiner") pod(joined, label="adder") - result = job.run() + _ = job.run() # Check operator node - joiner = result.pipeline.nodes["joiner"] + joiner = job.pipeline.nodes["joiner"] assert joiner.node_identity_path[0] == "Join" # Check function node - adder = result.pipeline.nodes["adder"] + adder = job.pipeline.nodes["adder"] assert adder.node_identity_path[0] == "add_values" @@ -416,13 +416,13 @@ def test_end_to_end_source_join_function(self, pipeline_db): assert isinstance(job.pipeline.nodes["joiner"], OperatorNode) assert isinstance(job.pipeline.nodes["adder"], FunctionNode) - result = job.run() + _ = job.run() - fn_records = result.nodes["adder"].get_all_records() + fn_records = job.nodes["adder"].get_all_records() assert fn_records is not None assert fn_records.num_rows == 2 - table = result.nodes["adder"].as_table() + table = job.nodes["adder"].as_table() totals = sorted(cast(list[int], table.column("total").to_pylist())) assert totals == [110, 220] @@ -444,8 +444,8 @@ def test_detached_pipeline_hash_differs_from_node(self, pipeline_db): with job: joined = Join()(src_a, src_b) pod(joined, label="adder") - result = job.run() - fn_node = result.nodes["adder"] + _ = job.run() + fn_node = job.nodes["adder"] derived = fn_node.as_source() # DerivedSource uses RootSource hash (schema-only), not topology hash assert derived.pipeline_hash() != fn_node.pipeline_hash() @@ -459,8 +459,8 @@ def test_detached_content_hash_differs_from_node(self, pipeline_db): with job: joined = Join()(src_a, src_b) pod(joined, label="adder") - result = job.run() - fn_node = result.nodes["adder"] + _ = job.run() + fn_node = job.nodes["adder"] derived = fn_node.as_source() assert derived.content_hash() != fn_node.content_hash() @@ -473,8 +473,8 @@ def test_two_derived_sources_from_same_node_share_pipeline_hash(self, pipeline_d with job: joined = Join()(src_a, src_b) pod(joined, label="adder") - result = job.run() - fn_node = result.nodes["adder"] + _ = job.run() + fn_node = job.nodes["adder"] derived1 = fn_node.as_source() derived2 = fn_node.as_source() assert derived1.pipeline_hash() == derived2.pipeline_hash() @@ -607,8 +607,8 @@ def test_compile_does_not_trigger_source_materialization(self, pipeline_db): assert isinstance(pre_run_node, FunctionNode) assert not isinstance(pre_run_node, FunctionJobNode) # Running should work correctly - result = job.run() - table = result.nodes["adder"].as_table() + _ = job.run() + table = job.nodes["adder"].as_table() assert table.num_rows == 2 @@ -675,8 +675,8 @@ def test_source_nodes_do_not_have_cache_path(self, pipeline_db): with job: joined = Join()(src_a, src_b) pod(joined, label="adder") - result = job.run() - records = result.nodes["adder"].get_all_records() + _ = job.run() + records = job.nodes["adder"].get_all_records() assert records is not None assert records.num_rows == 2 source_nodes = [ @@ -698,8 +698,8 @@ def test_pipeline_with_cached_source_input(self, pipeline_db): with job: joined = Join()(cached_a, cached_b) pod(joined, label="adder") - result = job.run() - records = result.nodes["adder"].get_all_records() + _ = job.run() + records = job.nodes["adder"].get_all_records() assert records is not None assert records.num_rows == 2 diff --git a/tests/test_pipeline/test_pipeline_job.py b/tests/test_pipeline/test_pipeline_job.py index 894ce29ab..a2a75b165 100644 --- a/tests/test_pipeline/test_pipeline_job.py +++ b/tests/test_pipeline/test_pipeline_job.py @@ -415,9 +415,9 @@ def test_run_executes_all_nodes(self, store): joined = Join()(src_a, src_b) pod(joined, label="adder") - result = job.run() + _ = job.run() - node = result.nodes["adder"] + node = job.nodes["adder"] records = node.get_all_records() assert records is not None assert records.num_rows == 2 @@ -432,9 +432,9 @@ def test_run_produces_correct_values(self, store): joined = Join()(src_a, src_b, label="joiner") pod(joined, label="adder") - result = job.run() + _ = job.run() - table = result.nodes["adder"].as_table() + table = job.nodes["adder"].as_table() totals = sorted(cast(list[int], table.column("total").to_pylist())) assert totals == [110, 220] # a: 10+100, b: 20+200 @@ -451,20 +451,24 @@ def test_run_partial_execution_skips_unbound_subgraph(self, store): joined = Join()(src_a, node_b) pod(joined, label="adder") - result = job.run() + _ = job.run() # Unbound sources should be reported - assert "spec_b" in result.unbound_sources + assert "spec_b" in job.unbound_sources + + def test_run_returns_its_summary_and_mutates_in_place(self, store): + """run() returns the run's summary, kept as last_run, and sets _has_run / _run_id.""" + from orcapod.pipeline import RunSummary - def test_run_returns_self_and_mutates_in_place(self, store): - """run() returns self and sets _has_run / _run_id in place.""" src_a, src_b = _make_two_sources() job = PipelineJob(store=store) with job: Join()(src_a, src_b, label="joiner") - result = job.run() - assert result is job + assert job.last_run is None + summary = job.run() + assert isinstance(summary, RunSummary) + assert job.last_run is summary assert job._has_run is True assert job._run_id is not None @@ -524,9 +528,9 @@ def test_run_twice_is_safe(self, store): pod(joined, label="adder") job.run() - result = job.run() # second run should not raise + _ = job.run() # second run should not raise - table = result.nodes["adder"].as_table() + table = job.nodes["adder"].as_table() totals = sorted(cast(list[int], table.column("total").to_pylist())) assert totals == [110, 220] @@ -545,8 +549,8 @@ def double_x(x: int) -> int: with job: pod(src, label="doubler") - completed = job.run(orchestrator=SyncPipelineOrchestrator()) - records = completed.nodes["doubler"].get_all_records() + _ = job.run(orchestrator=SyncPipelineOrchestrator()) + records = job.nodes["doubler"].get_all_records() assert records is not None assert records.num_rows == 2 @@ -565,8 +569,8 @@ def double_x(x: int) -> int: with job: pod(src, label="doubler") - completed = job.run(orchestrator=AsyncPipelineOrchestrator()) - records = completed.nodes["doubler"].get_all_records() + _ = job.run(orchestrator=AsyncPipelineOrchestrator()) + records = job.nodes["doubler"].get_all_records() assert records is not None assert records.num_rows == 2 @@ -586,13 +590,13 @@ def test_end_to_end_source_join_function(self, store): assert isinstance(job.pipeline.nodes["joiner"], OperatorNode) assert isinstance(job.pipeline.nodes["adder"], FunctionNode) - result = job.run() + _ = job.run() - fn_records = result.nodes["adder"].get_all_records() + fn_records = job.nodes["adder"].get_all_records() assert fn_records is not None assert fn_records.num_rows == 2 - table = result.nodes["adder"].as_table() + table = job.nodes["adder"].as_table() totals = sorted(cast(list[int], table.column("total").to_pylist())) assert totals == [110, 220] @@ -618,9 +622,9 @@ def test_from_pipeline_then_run(self, store): sources={"src_a": src_a, "src_b": src_b}, store=store, ) - result = job.run() + _ = job.run() - table = result.nodes["adder"].as_table() + table = job.nodes["adder"].as_table() totals = sorted(cast(list[int], table.column("total").to_pylist())) assert totals == [110, 220] @@ -632,7 +636,7 @@ def test_save_creates_file(self, store, tmp_path): with job: Join()(src_a, src_b) - path = tmp_path / "job.json" + path = tmp_path / "completed.json" job.save(str(path)) assert path.exists() @@ -642,7 +646,7 @@ def test_save_has_version(self, store, tmp_path): with job: Join()(src_a, src_b) - path = tmp_path / "job.json" + path = tmp_path / "completed.json" job.save(str(path)) data = json.loads(path.read_text()) assert data["orcapod_pipeline_job_version"] == "0.1.0" @@ -654,7 +658,7 @@ def test_save_includes_run_block(self, store, tmp_path): with job: Join()(src_a, src_b) - path = tmp_path / "job.json" + path = tmp_path / "completed.json" job.save(str(path)) data = json.loads(path.read_text()) assert data["run"]["status"] == "pending" @@ -667,7 +671,7 @@ def test_load_roundtrip_restores_pipeline(self, store, tmp_path): with job: Join()(src_a, src_b, label="joiner") - path = tmp_path / "job.json" + path = tmp_path / "completed.json" job.save(str(path)) loaded = PipelineJob.load(str(path)) @@ -684,9 +688,9 @@ def test_load_roundtrip_after_run(self, store, tmp_path): joined = Join()(src_a, src_b) pod(joined, label="adder") - completed = job.run() + _ = job.run() path = tmp_path / "completed.json" - completed.save(str(path)) + job.save(str(path)) # Verify the file has correct status data = json.loads(path.read_text()) @@ -730,9 +734,9 @@ def test_load_after_run_restores_has_run_true(self, store, tmp_path): with job: joined = Join()(src_a, src_b) pod(joined, label="adder") - completed = job.run() + _ = job.run() path = tmp_path / "completed.json" - completed.save(str(path)) + job.save(str(path)) loaded = PipelineJob.load(str(path), store=store) assert loaded._has_run is True @@ -766,10 +770,10 @@ def test_load_after_partial_run_preserves_unbound_sources(self, store, tmp_path) with job: joined = Join()(src_a, node_b) pod(joined, label="adder") - result = job.run() - assert "unbound_b" in result.unbound_sources + _ = job.run() + assert "unbound_b" in job.unbound_sources path = tmp_path / "partial.json" - result.save(str(path)) + job.save(str(path)) loaded = PipelineJob.load(str(path), store=store) assert "unbound_b" in loaded.unbound_sources @@ -783,7 +787,7 @@ def test_load_bind_works_after_load(self, store, tmp_path): joined = Join()(src_a, src_b) pod(joined, label="adder") - path = tmp_path / "job.json" + path = tmp_path / "completed.json" job.save(str(path)) # Load without sources so bind() is needed. @@ -807,7 +811,7 @@ def test_load_is_not_runnable_after_bind(self, store, tmp_path): joined = Join()(src_a, src_b) pod(joined, label="adder") - path = tmp_path / "job.json" + path = tmp_path / "completed.json" job.save(str(path)) loaded = PipelineJob.load(str(path)) @@ -833,9 +837,9 @@ def test_load_nodes_are_job_nodes(self, store, tmp_path): with job: joined = Join()(src_a, src_b) pod(joined, label="adder") - job.save(str(tmp_path / "job.json")) + job.save(str(tmp_path / "completed.json")) - loaded = PipelineJob.load(str(tmp_path / "job.json"), store=store) + loaded = PipelineJob.load(str(tmp_path / "completed.json"), store=store) adder = loaded.nodes.get("adder") assert adder is not None, "Expected 'adder' in loaded job nodes" assert isinstance(adder, FunctionJobNode), ( @@ -852,12 +856,12 @@ def test_load_get_all_records_returns_results(self, store, tmp_path): joined = Join()(src_a, src_b) pod(joined, label="adder") - completed = job.run() - original_records = completed.nodes["adder"].get_all_records() + _ = job.run() + original_records = job.nodes["adder"].get_all_records() assert original_records is not None, "run() produced no records" path = tmp_path / "completed.json" - completed.save(str(path)) + job.save(str(path)) loaded = PipelineJob.load(str(path), store=store) loaded_records = loaded.nodes["adder"].get_all_records() diff --git a/tests/test_pipeline/test_serialization.py b/tests/test_pipeline/test_serialization.py index e58952f13..59c2748f2 100644 --- a/tests/test_pipeline/test_serialization.py +++ b/tests/test_pipeline/test_serialization.py @@ -171,12 +171,12 @@ def test_load_bindable_and_runnable(self, spec_pipeline): job = PipelineJob.from_pipeline( loaded, sources={"source_a": src_a, "source_b": src_b}, store=store ) - completed = job.run() - assert completed._has_run is True + _ = job.run() + assert job._has_run is True # Verify all source specs were resolved (no unresolved specs) - assert completed.unbound_sources == [] + assert job.unbound_sources == [] # Verify the join node is present in the compiled pipeline - joiner_node = completed.pipeline.nodes.get("joiner") + joiner_node = job.pipeline.nodes.get("joiner") assert joiner_node is not None def test_load_hash_graph_has_node_types(self, spec_pipeline): diff --git a/tests/test_pipeline/test_store_write_failure.py b/tests/test_pipeline/test_store_write_failure.py index 066737fed..24efce965 100644 --- a/tests/test_pipeline/test_store_write_failure.py +++ b/tests/test_pipeline/test_store_write_failure.py @@ -755,10 +755,10 @@ def on_data_crash(self, node_label, tag, data, error) -> None: self.crashes.append((node_label, str(tag["k"]))) -# The per-row handlers' log messages: sync ``FunctionNode.execute`` and async -# ``process_one`` (DEBUG there, ITL-879). +# The per-row handlers' log messages, both at DEBUG: sync ``FunctionNode.execute`` +# (ITL-997) and async ``process_one`` (ITL-879). ROW_FAILURE_LOGS = ( - "Data execution failed in %s: %s", + "Data execution failed in %s, skipping: %s", "Data processing failed, skipping: %s", ) From f28bfab94378d4e2f2b3a3c14e1813729cdfe7f2 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 17:36:22 -0800 Subject: [PATCH 4/6] fix(pipeline): address review of the bridged async run and the run summary (ITL-994, ITL-997) - An interrupt during the bridge thread's start is handled: a lock-guarded cancelled flag keeps a cancelled run from starting its task. - A second interrupt while a cancelled run waits for a function returns at once, with a RuntimeWarning naming the node. For a sync function nothing more is written when it returns (tested on memory and Delta stores); for an async function that blocks the event loop the warning says rows in progress may still be stored. - The traceback pointer quotes the node label as a valid string literal. - Docstrings for the new helpers and tests; docs and changelog describe the two interrupt presses. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-994.md | 7 +- docs/concepts/function-pods.md | 14 +- src/orcapod/pipeline/async_orchestrator.py | 269 +++++++++++++++--- src/orcapod/pipeline/run_summary.py | 10 +- .../plans/2026-11-12-itl-994-997-job-run.md | 5 + .../test_job_run_in_event_loop.py | 203 +++++++++++++ tests/test_pipeline/test_job_run_summary.py | 25 ++ 7 files changed, 491 insertions(+), 42 deletions(-) diff --git a/changelog.d/ITL-994.md b/changelog.d/ITL-994.md index 9ebba2215..ddcd23d1e 100644 --- a/changelog.d/ITL-994.md +++ b/changelog.d/ITL-994.md @@ -12,5 +12,8 @@ `RuntimeError: asyncio.run() cannot be called from a running event loop`. Inside a loop, the run now goes on a separate thread while the call waits, so the same line works in a script and in a notebook; it blocks that loop until the run ends (use `await job.run_async()` in - async code). The interrupt button stops the run and nothing keeps running after it. Stored - rows, record ids and hashes are the same as a run outside a loop. (ITL-994) + async code). The interrupt button cancels the run and waits for functions in the middle of + a row to return; pressing it again stops waiting at once, with a warning naming the node + whose function has not returned. Nothing more is written to the store when it returns (an + `async def` function that blocks the event loop is the exception, and the warning says + so). Stored rows, record ids and hashes are the same as a run outside a loop. (ITL-994) diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index a86194448..2c775891d 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -426,8 +426,18 @@ Both orchestrators store the same rows, with the same record ids and hashes. A notebook kernel already runs an event loop. There, `job.run(...)` runs the job on a separate thread and waits for it, so the cell blocks until the run ends, as it does with the -sync orchestrator. The interrupt button stops the run: rows already stored stay stored, and -the next run computes only the rest. +sync orchestrator. The interrupt button cancels the run: rows already stored stay stored, and +the next run computes only the rest. A function that is in the middle of a row cannot be +interrupted, so the cell waits until it returns. If it does not return (a function that +loops forever, say), press the interrupt button again: the cell stops waiting at once, with a +warning that the function in that node has not returned yet. The run is already cancelled, +so nothing more is written to the store when the function does return. To stop the function +itself, restart the kernel. + +An `async def` function must not block (with `time.sleep`, a busy loop or another blocking +call). It runs on the event loop's own thread, so while it blocks, the cancellation cannot +run. In that case the warning says so: rows already in progress may still be stored when +the function returns, before the run stops. In async code, await the run instead, so the event loop stays free while it runs: `await job.run_async()` (the async orchestrator is its default). This also works at the top diff --git a/src/orcapod/pipeline/async_orchestrator.py b/src/orcapod/pipeline/async_orchestrator.py index a6913c832..979ceaab2 100644 --- a/src/orcapod/pipeline/async_orchestrator.py +++ b/src/orcapod/pipeline/async_orchestrator.py @@ -13,6 +13,7 @@ import logging import threading import uuid +import warnings from collections import defaultdict from collections.abc import Callable, Coroutine from typing import TYPE_CHECKING, Any, Literal, TypeVar @@ -83,7 +84,10 @@ def run( while the caller waits, so the call works the same in both places. The caller's loop is blocked until the run ends; async code should await ``run_async`` instead. A ``KeyboardInterrupt`` while waiting (the notebook's interrupt button) - cancels the run, waits for it to stop, and is re-raised. + cancels the run, waits for the functions still running to return, and is + re-raised. A second interrupt during that wait returns at once, with a + ``RuntimeWarning`` naming the nodes whose functions have not returned; when they + do, nothing more is written to the store. Args: graph: A ``GraphProtocol`` DAG with GraphNode objects as vertices. @@ -99,26 +103,44 @@ def run( Returns: OrchestratorResult with node outputs. """ + try: + _ = asyncio.get_running_loop() + except RuntimeError: + loop_running = False + else: + loop_running = True + # Run outside the handler, so the run's own errors carry no context from it. + if not loop_running: + return asyncio.run( + self._run_async( + graph, + materialize_results, + observer=observer, + run_id=run_id, + pipeline_uri=pipeline_uri, + ) + ) + + from orcapod.pipeline.composite_observer import CompositeObserver + + rows_in_flight = _RowsInFlight(_async_function_labels(graph)) + watched: Any = ( + rows_in_flight + if observer is None + else CompositeObserver(observer, rows_in_flight) + ) def make_run() -> Coroutine[Any, Any, OrchestratorResult]: + """Build the run's coroutine, observed for rows still in flight.""" return self._run_async( graph, materialize_results, - observer=observer, + observer=watched, run_id=run_id, pipeline_uri=pipeline_uri, ) - try: - _ = asyncio.get_running_loop() - except RuntimeError: - loop_running = False - else: - loop_running = True - # Run outside the handler, so the run's own errors carry no context from it. - if loop_running: - return _run_on_own_thread(make_run) - return asyncio.run(make_run()) + return _run_on_own_thread(make_run, rows_in_flight) async def run_async( self, @@ -312,40 +334,55 @@ async def _run_async( effective_observer.on_run_end(run_id) -def _run_on_own_thread(make_run: Callable[[], Coroutine[Any, Any, _T]]) -> _T: +def _run_on_own_thread( + make_run: Callable[[], Coroutine[Any, Any, _T]], rows_in_flight: _RowsInFlight +) -> _T: """Run a coroutine to completion on a new thread with its own event loop. For a caller whose thread already runs an event loop, where ``asyncio.run`` is refused. The caller blocks until the run ends and gets its result, or its exception - unchanged. The thread runs in a copy of the caller's context variables. A - ``KeyboardInterrupt`` raised in the caller while it waits cancels the run, waits for - the run to stop (functions already running in the loop's thread pool finish their - current row), and is re-raised, so no run keeps writing after the caller stopped. + unchanged. The thread runs in a copy of the caller's context variables. + + A first ``KeyboardInterrupt`` in the caller (the notebook's interrupt button) + cancels the run and waits for it to stop: a function already running on the loop's + thread pool cannot be interrupted, so the wait lasts until it returns. A second + interrupt during that wait returns control at once: it warns that a function has + not returned yet, naming the nodes with rows in flight, and is raised. The cancelled + run never resumes, so when that function returns nothing more is written. Args: - make_run: Builds the coroutine; called on the new thread. + make_run: Builds the coroutine; called on the new thread, and not at all when + the caller is interrupted before the thread starts it. + rows_in_flight: Observes the run, to name in the warning the nodes whose + functions have not returned. Returns: What the coroutine returned. """ loop = asyncio.new_event_loop() + # Guards the hand-over between caller and thread: once the caller has cancelled, + # the thread does not start the run; once the thread has started it, the caller + # cancels the task. + state_lock = threading.Lock() + cancelled = threading.Event() tasks: list[asyncio.Task[_T]] = [] - task_created = threading.Event() finished = threading.Event() results: list[_T] = [] errors: list[BaseException] = [] def run_loop() -> None: + """Run the coroutine on this thread's loop, unless the caller cancelled first.""" asyncio.set_event_loop(loop) try: - task = loop.create_task(make_run()) - tasks.append(task) - task_created.set() + with state_lock: + if cancelled.is_set(): + return + task = loop.create_task(make_run()) + tasks.append(task) results.append(loop.run_until_complete(task)) except BaseException as exc: # noqa: BLE001 # handed to the waiting caller errors.append(exc) finally: - task_created.set() try: loop.run_until_complete(loop.shutdown_asyncgens()) loop.run_until_complete(loop.shutdown_default_executor()) @@ -358,30 +395,188 @@ def run_loop() -> None: thread = threading.Thread( target=context.run, args=(run_loop,), name="orcapod-async-run", daemon=True ) - thread.start() - # Wait on an event, not Thread.join: on Python 3.12 an interrupt inside join marks - # a thread that is still running as stopped, and later joins return at once. try: - # Wait in short slices: a pending interrupt is raised between them even when - # the signal reached another thread and did not wake this one's wait. - while not finished.wait(_INTERRUPT_CHECK_SECONDS): - pass + thread.start() + _wait_until(finished) except KeyboardInterrupt: - _ = task_created.wait() - if tasks: + with state_lock: + cancelled.set() + started = tasks[0] if tasks else None + if started is not None: # The loop may have closed after the run ended on its own. with contextlib.suppress(RuntimeError): - _ = loop.call_soon_threadsafe(tasks[0].cancel) - # A second interrupt here stops the wait; the cancelled run then winds down - # on its own thread. - while not finished.wait(_INTERRUPT_CHECK_SECONDS): - pass + _ = loop.call_soon_threadsafe(started.cancel) + try: + _wait_until(finished) + except KeyboardInterrupt: + _warn_function_still_running( + rows_in_flight.labels(), rows_in_flight.async_labels + ) + raise raise if errors: raise errors[0] return results[0] +def _wait_until(event: threading.Event) -> None: + """Block until ``event`` is set, letting a ``KeyboardInterrupt`` through. + + Waits on an event, not ``Thread.join``: on Python 3.12 an interrupt inside join + marks a thread that is still running as stopped. Waits in short slices, so a pending + interrupt is raised between them even when the signal reached another thread. + + Args: + event: The event to wait for. + """ + while not event.wait(_INTERRUPT_CHECK_SECONDS): + pass + + +def _warn_function_still_running( + labels: list[str], async_labels: frozenset[str] +) -> None: + """Warn that a cancelled run's function has not returned, naming its nodes. + + A sync function runs on the loop's thread pool: the run is already cancelled, so + when it returns nothing more is written. An async function that has not returned is + blocking the loop's own thread, so the cancellation cannot run until it returns, and + rows already in progress may still be stored then. + + Args: + labels: The nodes with rows whose function has not returned. + async_labels: The nodes whose function is an async function. + """ + blocking = [label for label in labels if label in async_labels] + if blocking: + message = ( + f"The run was cancelled, but an async function in {_nodes(blocking)} is " + "blocking the event loop and has not returned yet. When it returns, rows " + "already in progress may still be stored before the run stops. To stop it " + "now, restart the kernel (or end the process)." + ) + else: + which = f"a function in {_nodes(labels)}" if labels else "a function" + message = ( + f"The run was cancelled, but {which} has not returned yet. Nothing more " + "will be written to the store when it does. To stop it, restart the kernel " + "(or end the process)." + ) + # Shown every time, whatever filters hide repeated warnings. + with warnings.catch_warnings(): + warnings.simplefilter("always", RuntimeWarning) + warnings.warn(message, RuntimeWarning, stacklevel=2) + + +def _nodes(labels: list[str]) -> str: + """``node 'a'`` or ``nodes 'a', 'b'``.""" + names = ", ".join(repr(label) for label in labels) + return f"node {names}" if len(labels) == 1 else f"nodes {names}" + + +def _async_function_labels(graph: GraphProtocol[Any]) -> frozenset[str]: + """The labels of the graph's function nodes whose function is an async function. + + Args: + graph: The run's graph. + + Returns: + The labels. + """ + labels: set[str] = set() + for node in graph.nodes(): + label: str = node.label + if not (is_function_node(node) or is_side_effect_function_node(node)): + continue + # Only a Python data function has a kind; a loaded node has no pod. + data_function = getattr(getattr(node, "producer", None), "data_function", None) + if getattr(data_function, "is_async", False) is True: + labels.add(label) + return frozenset(labels) + + +class _RowsInFlight: + """Observes a run: per node label, the rows started whose function has not returned. + + Rows already handed to the thread pool but not started are cancelled without an + end event, so a node can count rows that were only queued. Every contextualized copy + is this observer itself. Hooks may be called from several threads, so the counts are + updated under a lock. + + Attributes: + async_labels: The nodes whose function is an async function. + """ + + def __init__(self, async_labels: frozenset[str]) -> None: + """Start with no rows in flight. + + Args: + async_labels: The nodes whose function is an async function. + """ + self.async_labels: frozenset[str] = async_labels + self._lock: threading.Lock = threading.Lock() + self._open: dict[str, int] = {} + + def labels(self) -> list[str]: + """The nodes with rows in flight, in the order their first row started. + + Returns: + The node labels. + """ + with self._lock: + return [label for label, n in self._open.items() if n > 0] + + def _change(self, node_label: str, delta: int) -> None: + """Add ``delta`` to the rows in flight of ``node_label``.""" + with self._lock: + self._open[node_label] = self._open.get(node_label, 0) + delta + + def contextualize(self, *identity_path: str) -> _RowsInFlight: + """Return this observer: rows are keyed by node label.""" + return self + + def on_run_start(self, run_id: str, pipeline_uri: str = "") -> None: + """Nothing to track.""" + + def on_run_end(self, run_id: str) -> None: + """Nothing to track.""" + + def on_node_start( + self, node_label: str, node_hash: str, tag_schema: Any = None + ) -> None: + """Nothing to track.""" + + def on_node_end(self, node_label: str, node_hash: str) -> None: + """Nothing to track.""" + + def on_data_start(self, node_label: str, tag: Any, data: Any) -> None: + """A row's function starts.""" + self._change(node_label, 1) + + def on_data_end( + self, + node_label: str, + tag: Any, + input_data: Any, + output_data: Any, + cached: bool, + ) -> None: + """A row's function returned (or its result came from the store).""" + self._change(node_label, -1) + + def on_data_crash( + self, node_label: str, tag: Any, data: Any, error: Exception + ) -> None: + """A row's function raised.""" + self._change(node_label, -1) + + def create_data_logger(self, tag: Any, data: Any) -> Any: + """No logging of its own.""" + from orcapod.pipeline.observer import NoOpLogger + + return NoOpLogger() + + def _start_node( tg: asyncio.TaskGroup, node: Any, work: Coroutine[Any, Any, None] ) -> None: diff --git a/src/orcapod/pipeline/run_summary.py b/src/orcapod/pipeline/run_summary.py index 1967d64e7..cf094c1c4 100644 --- a/src/orcapod/pipeline/run_summary.py +++ b/src/orcapod/pipeline/run_summary.py @@ -8,6 +8,7 @@ from __future__ import annotations import html +import json import threading import traceback from dataclasses import dataclass @@ -119,6 +120,7 @@ def print_traceback(self, label: str | None = None) -> None: print(f"No rows failed{where} in this run.") def _headline(self) -> str: + """The first line: the job and whether any row failed.""" if self.failed_rows: outcome = f"{_rows(self.failed_rows)} failed." else: @@ -126,6 +128,7 @@ def _headline(self) -> str: return f"Run of job {self.job_name!r}: {outcome}" def __repr__(self) -> str: + """The summary as text: one line per node, and the first error of a failing one.""" lines = [self._headline()] for node in self.nodes.values(): lines.append( @@ -176,6 +179,7 @@ def _repr_html_(self) -> str: def _rows(n: int) -> str: + """``1 row`` or ``n rows``.""" return f"{n} row" if n == 1 else f"{n} rows" @@ -192,7 +196,11 @@ def _error_line(error: BaseException | None) -> str: def _traceback_call(label: str) -> str: - return f'job.last_run.print_traceback("{label}")' + """The call that prints a node's traceback, with the label as a valid string literal. + + A JSON string is a valid Python string literal, and keeps double quotes. + """ + return f"job.last_run.print_traceback({json.dumps(label)})" class RunCountingObserver: diff --git a/superpowers/plans/2026-11-12-itl-994-997-job-run.md b/superpowers/plans/2026-11-12-itl-994-997-job-run.md index 0e31e2c14..3d9966311 100644 --- a/superpowers/plans/2026-11-12-itl-994-997-job-run.md +++ b/superpowers/plans/2026-11-12-itl-994-997-job-run.md @@ -47,6 +47,11 @@ fragment per issue. orcapod.errors, job.last_run, RunCountingObserver composed with the user's observer, sync per-row log at DEBUG; 38 test sites updated; empty-input node reports zeros (coordinator ask). Full suite before merge: 10107 passed, 116 skipped, 23 xfailed. +- PR #337 opened. CodeRabbit round 1 (review of a05064ce): plan approved by coordinator + (items 1 and 4 declined; 2, 5, nitpicks accepted; 3 changed: second interrupt returns at + once with a RuntimeWarning, per-case wording for sync vs async functions, proven by tests). + Flaky CI test seen: test_job_crash_recovery::test_crash_mid_batch_keeps_committed_batches[async] + (coordinator records it on ITL-1001). ## Findings (today, main 290889e8) diff --git a/tests/test_pipeline/test_job_run_in_event_loop.py b/tests/test_pipeline/test_job_run_in_event_loop.py index 299da32f4..afb88db4d 100644 --- a/tests/test_pipeline/test_job_run_in_event_loop.py +++ b/tests/test_pipeline/test_job_run_in_event_loop.py @@ -9,7 +9,9 @@ import _thread import asyncio +import threading import time +import warnings from collections.abc import Callable from pathlib import Path from typing import Any @@ -40,6 +42,7 @@ def run_inside_running_loop(job: PipelineJob) -> None: """ async def notebook_cell() -> None: + """A notebook cell: the job run inside the kernel's running loop.""" job.run(orchestrator=AsyncPipelineOrchestrator()) asyncio.run(notebook_cell()) @@ -71,6 +74,7 @@ def stored_rows(job: PipelineJob) -> tuple[list[dict[str, Any]], list[dict[str, table = job.nodes["doubled"].as_table(all_info=True) def by_tag(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """The rows sorted by their tag ``k``.""" return sorted(rows, key=lambda row: row["k"]) return by_tag(records.to_pylist()), by_tag(table.to_pylist()) @@ -80,6 +84,7 @@ def by_tag(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: def test_run_inside_running_loop_stores_what_a_script_run_stores( store_kind: str, tmp_path: Path ) -> None: + """A run from inside a running loop stores the same rows, ids and hashes as a script run.""" script_job = make_job(STORES[store_kind](tmp_path / "script")) script_job.run(orchestrator=AsyncPipelineOrchestrator()) @@ -92,6 +97,7 @@ def test_run_inside_running_loop_stores_what_a_script_run_stores( def test_rerun_inside_running_loop_reuses_stored_rows(tmp_path: Path) -> None: + """A second run inside a running loop reads the stored rows and changes none of them.""" store = DeltaTableDatabase(tmp_path / "store") first_job = make_job(store) run_inside_running_loop(first_job) @@ -111,18 +117,23 @@ class FailingWrites: """A store whose every write fails.""" def __init__(self, inner: Any) -> None: + """Wrap ``inner``.""" self._inner = inner def at(self, *path_components: str) -> FailingWrites: + """A view whose writes fail too.""" return FailingWrites(self._inner.at(*path_components)) def add_record(self, *args: Any, **kwargs: Any) -> Any: + """Fail.""" raise OSError("disk unavailable") def add_records(self, *args: Any, **kwargs: Any) -> Any: + """Fail.""" raise OSError("disk unavailable") def __getattr__(self, name: str) -> Any: + """Delegate everything else.""" return getattr(self._inner, name) job = make_job(FailingWrites(InMemoryArrowDatabase())) @@ -134,6 +145,7 @@ def test_run_inside_running_loop_returns_to_the_loop(tmp_path: Path) -> None: """The loop that called the run keeps working after it (a notebook's next cell).""" async def two_cells() -> int: + """Run a job, then use the loop again, as the next cell would.""" job = make_job(InMemoryArrowDatabase()) job.run(orchestrator=AsyncPipelineOrchestrator()) await asyncio.sleep(0) @@ -146,12 +158,14 @@ async def two_cells() -> int: def test_await_run_async_stores_what_a_script_run_stores( store_kind: str, tmp_path: Path ) -> None: + """``await job.run_async()`` stores the same rows, ids and hashes as a script run.""" script_job = make_job(STORES[store_kind](tmp_path / "script")) script_job.run(orchestrator=AsyncPipelineOrchestrator()) async_job = make_job(STORES[store_kind](tmp_path / "async")) async def async_code() -> None: + """Async code awaiting the run.""" await async_job.run_async() asyncio.run(async_code()) @@ -160,6 +174,7 @@ async def async_code() -> None: def test_run_async_refuses_the_sync_orchestrator() -> None: + """``run_async`` with a sync orchestrator raises ``TypeError`` naming ``job.run``.""" job = make_job(InMemoryArrowDatabase()) with pytest.raises(TypeError, match=r"job\.run"): _ = asyncio.run( @@ -193,6 +208,7 @@ def test_interrupt_while_running_inside_a_loop_stops_the_run() -> None: slow_and_interrupted.pod(source, label="slow") async def notebook_cell() -> None: + """A notebook cell: the job run inside the kernel's running loop.""" job.run(orchestrator=AsyncPipelineOrchestrator()) # A bare loop, as a kernel has: asyncio.run would install its own SIGINT handler. @@ -207,3 +223,190 @@ async def notebook_cell() -> None: assert calls_when_stopped < N_SLOW_ROWS time.sleep(0.2) assert len(slow_calls) == calls_when_stopped + + +STORE_WRITES = ("add_record", "add_records", "flush", "flush_batch", "set_metadata") + + +class CountingWrites: + """A store that records the name of every write or commit made through it.""" + + def __init__(self, inner: Any, writes: list[str]) -> None: + """Wrap ``inner``; every view appends to the same ``writes``.""" + self._inner = inner + self._writes = writes + + def at(self, *path_components: str) -> CountingWrites: + """A view recording into the same list.""" + return CountingWrites(self._inner.at(*path_components), self._writes) + + def __getattr__(self, name: str) -> Any: + """Delegate; record the calls of the write methods.""" + attr = getattr(self._inner, name) + if name not in STORE_WRITES: + return attr + + def recorded(*args: Any, **kwargs: Any) -> Any: + """Record the write, then make it.""" + self._writes.append(name) + return attr(*args, **kwargs) + + return recorded + + +stuck_release = threading.Event() +press_interrupt = threading.Event() +stuck_calls: list[int] = [] +after_calls: list[int] = [] + + +@function_pod(output_keys="y") +def stuck_on_first_row(x: int) -> int: + """Row 0 presses the interrupt button twice and does not return until released.""" + stuck_calls.append(x) + if x == 0 and press_interrupt.is_set(): + _thread.interrupt_main() # first press: cancels the run, which then waits + # Second press, while the run waits for this function to return. + threading.Timer(1.0, _thread.interrupt_main).start() + _ = stuck_release.wait(timeout=30) + return x + + +@function_pod(output_keys="z") +def after_stuck(y: int) -> int: + """Count the rows that reach the next node.""" + after_calls.append(y) + return y + 1 + + +def make_stuck_job(store: Any) -> PipelineJob: + """Six rows through ``stuck_on_first_row`` and then ``after_stuck``.""" + rows = [{"k": f"r{i}", "x": i} for i in range(6)] + job = PipelineJob(name="stuck_demo", store=store) + with job: + source = DictSource(data=rows, tag_columns=["k"], source_id="rows") + stuck = stuck_on_first_row.pod(source, label="stuck") + after_stuck.pod(stuck, label="after") + return job + + +def bridge_threads() -> list[threading.Thread]: + """The threads still running a job's async run.""" + return [ + t + for t in threading.enumerate() + if t.name == "orcapod-async-run" and t.is_alive() + ] + + +@pytest.mark.parametrize("store_kind", list(STORES)) +def test_second_interrupt_returns_and_a_late_function_writes_nothing( + store_kind: str, tmp_path: Path +) -> None: + """A second interrupt returns at once; the function returning later stores nothing. + + The first interrupt cancels the run, which then waits for row 0's function, still + running on a thread where it cannot be interrupted. The second interrupt returns + control with a warning naming the node. When the function returns afterwards, no + store write happens and no row reaches the next node, and row 0's result is never + stored: a later run computes it. + """ + stuck_release.clear() + press_interrupt.set() + stuck_calls.clear() + after_calls.clear() + inner = STORES[store_kind](tmp_path / "store") + writes: list[str] = [] + job = make_stuck_job(CountingWrites(inner, writes)) + + async def notebook_cell() -> None: + """A notebook cell: the job run inside the kernel's running loop.""" + job.run(orchestrator=AsyncPipelineOrchestrator()) + + # A bare loop, as a kernel has: asyncio.run would install its own SIGINT handler. + loop = asyncio.new_event_loop() + try: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + with pytest.raises(KeyboardInterrupt): + loop.run_until_complete(notebook_cell()) + finally: + loop.close() + + messages = [str(w.message) for w in caught if w.category is RuntimeWarning] + assert len(messages) == 1, messages + assert "but a function in node 'stuck' has not returned yet" in messages[0] + assert "Nothing more will be written to the store" in messages[0] + assert "restart the kernel" in messages[0] + assert bridge_threads(), "control returned before the stuck function did" + + writes_when_returned = list(writes) + rows_after_when_returned = list(after_calls) + stuck_release.set() + deadline = time.monotonic() + 30 + while bridge_threads() and time.monotonic() < deadline: + time.sleep(0.05) + assert not bridge_threads() + + assert writes == writes_when_returned + assert after_calls == rows_after_when_returned + + # Row 0's result was never stored: a later run on the same store computes it. + inner.flush() + stuck_calls.clear() + press_interrupt.clear() + rerun = make_stuck_job(inner) + _ = rerun.run() + assert 0 in stuck_calls + + +@function_pod(output_keys="y") +async def async_stuck_on_first_row(x: int) -> int: + """Row 0 presses the interrupt button twice and blocks the event loop until released.""" + if x == 0: + _thread.interrupt_main() + threading.Timer(1.0, _thread.interrupt_main).start() + # A blocking wait inside an async function: the bug that blocks the loop. + _ = stuck_release.wait(timeout=30) + return x + + +def test_second_interrupt_names_an_async_function_blocking_the_loop() -> None: + """For an async function blocking the loop, the warning says rows may still be stored. + + The cancellation cannot run on a loop whose thread the function holds, so rows + already in progress may be stored once it returns; the warning says so instead of + promising that nothing more is written. The run still stops once it returns. + """ + stuck_release.clear() + rows = [{"k": f"r{i}", "x": i} for i in range(6)] + job = PipelineJob(name="async_stuck_demo", store=InMemoryArrowDatabase()) + with job: + source = DictSource(data=rows, tag_columns=["k"], source_id="rows") + stuck = async_stuck_on_first_row.pod(source, label="stuck") + after_stuck.pod(stuck, label="after") + + async def notebook_cell() -> None: + """A notebook cell: the job run inside the kernel's running loop.""" + job.run(orchestrator=AsyncPipelineOrchestrator()) + + loop = asyncio.new_event_loop() + try: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + with pytest.raises(KeyboardInterrupt): + loop.run_until_complete(notebook_cell()) + finally: + loop.close() + + messages = [str(w.message) for w in caught if w.category is RuntimeWarning] + assert len(messages) == 1, messages + assert "an async function in node 'stuck' is blocking the event loop" in messages[0] + assert "rows already in progress may still be stored" in messages[0] + assert "Nothing more will be written" not in messages[0] + + stuck_release.set() + deadline = time.monotonic() + 30 + while bridge_threads() and time.monotonic() < deadline: + time.sleep(0.05) + assert not bridge_threads() diff --git a/tests/test_pipeline/test_job_run_summary.py b/tests/test_pipeline/test_job_run_summary.py index 48062cc68..ef120feff 100644 --- a/tests/test_pipeline/test_job_run_summary.py +++ b/tests/test_pipeline/test_job_run_summary.py @@ -8,6 +8,7 @@ from __future__ import annotations +import ast import warnings from collections.abc import Callable from pathlib import Path @@ -96,15 +97,18 @@ def make_job(store: Any, pod: Any, label: str, n_rows: int = 4) -> PipelineJob: @pytest.fixture(params=list(ORCHESTRATORS)) def orchestrator(request: pytest.FixtureRequest) -> Any: + """Each orchestrator, freshly built.""" return ORCHESTRATORS[request.param]() @pytest.fixture(params=list(STORES)) def store(request: pytest.FixtureRequest, tmp_path: Path) -> Any: + """Each store kind, empty.""" return STORES[request.param](tmp_path / "store") def test_some_rows_fail(store: Any, orchestrator: Any) -> None: + """Rows that fail are counted and reported once; the others are computed and stored.""" job = make_job(store, even_only, "evens") with pytest.warns(RowFailuresWarning) as caught: summary = run(job, orchestrator) @@ -125,6 +129,7 @@ def test_some_rows_fail(store: Any, orchestrator: Any) -> None: def test_every_row_fails(store: Any, orchestrator: Any) -> None: + """A function failing on every row is reported with its error, not as an empty success.""" job = make_job(store, always_fails, "broken") with pytest.warns(RowFailuresWarning, match="4 of 4 rows failed in node 'broken'"): summary = run(job, orchestrator) @@ -140,6 +145,7 @@ def test_every_row_fails(store: Any, orchestrator: Any) -> None: def test_clean_run_is_quiet(store: Any, orchestrator: Any) -> None: + """A run without failed rows issues no warning and counts every row as computed.""" job = make_job(store, double, "doubled") with warnings.catch_warnings(): warnings.simplefilter("error") @@ -152,6 +158,7 @@ def test_clean_run_is_quiet(store: Any, orchestrator: Any) -> None: def test_rerun_counts_rows_from_the_store(store: Any, orchestrator: Any) -> None: + """Running again counts the stored rows as from the store and computes only the new row.""" run(make_job(store, double, "doubled"), orchestrator) summary = run(make_job(store, double, "doubled", n_rows=5), orchestrator) @@ -159,6 +166,7 @@ def test_rerun_counts_rows_from_the_store(store: Any, orchestrator: Any) -> None def test_fail_fast_still_raises(store: Any) -> None: + """``error_policy='fail_fast'`` raises the row's error as before.""" job = make_job(store, even_only, "evens") with pytest.raises(ValueError, match="odd x="): job.run(orchestrator=SyncPipelineOrchestrator(error_policy="fail_fast")) @@ -167,6 +175,7 @@ def test_fail_fast_still_raises(store: Any) -> None: def test_print_traceback_shows_the_function_frame( capsys: pytest.CaptureFixture[str], ) -> None: + """``print_traceback`` prints the traceback down to the user's function.""" job = make_job(InMemoryArrowDatabase(), always_fails, "broken") with pytest.warns(RowFailuresWarning): run(job, SyncPipelineOrchestrator()) @@ -180,6 +189,7 @@ def test_print_traceback_shows_the_function_frame( def test_notebook_representation_names_the_failing_node() -> None: + """The notebook table names the failing node, its error type and message.""" job = make_job(InMemoryArrowDatabase(), even_only, "evens") with pytest.warns(RowFailuresWarning): summary = run(job, SyncPipelineOrchestrator()) @@ -191,6 +201,7 @@ def test_notebook_representation_names_the_failing_node() -> None: def test_run_async_returns_the_same_summary(store: Any) -> None: + """``run_async`` returns the summary, warns, and keeps it as ``last_run``.""" import asyncio job = make_job(store, even_only, "evens") @@ -223,3 +234,17 @@ def test_node_with_empty_input_counts_zero_rows(store: Any, orchestrator: Any) - assert node_counts(summary, "broken") == (0, 0, 4) assert node_counts(summary, "after") == (0, 0, 0) assert "after: 0 rows: 0 computed, 0 from store, 0 failed" in repr(summary) + + +def test_traceback_pointer_is_valid_python_for_any_label() -> None: + """A label with quotes still gives a call that runs and prints the traceback.""" + label = "weight\"2's" + job = make_job(InMemoryArrowDatabase(), always_fails, label) + with pytest.warns(RowFailuresWarning) as caught: + summary = run(job, SyncPipelineOrchestrator()) + + pointer = repr(summary).splitlines()[-1].split("Full traceback: ", 1)[1] + assert pointer in str(caught[0].message) + call = ast.parse(pointer, mode="eval").body + assert isinstance(call, ast.Call) + assert ast.literal_eval(call.args[0]) == label From f5325231decb8430bf5ca0cacf4fc0794dfe4b30 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 17:47:50 -0800 Subject: [PATCH 5/6] test(pipeline): leave per-run source-info columns out of the node table comparison (ITL-994) After #335 a function node's table with all info carries the output's source-info column, which holds the per-run result id and differs between any two runs. Co-Authored-By: Claude Opus 5.5 --- tests/test_pipeline/test_job_run_in_event_loop.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_pipeline/test_job_run_in_event_loop.py b/tests/test_pipeline/test_job_run_in_event_loop.py index afb88db4d..264013e0b 100644 --- a/tests/test_pipeline/test_job_run_in_event_loop.py +++ b/tests/test_pipeline/test_job_run_in_event_loop.py @@ -72,6 +72,7 @@ def stored_rows(job: PipelineJob) -> tuple[list[dict[str, Any]], list[dict[str, [c for c in PER_RUN_COLUMNS if c in records.column_names] ) table = job.nodes["doubled"].as_table(all_info=True) + table = table.drop_columns([c for c in PER_RUN_COLUMNS if c in table.column_names]) def by_tag(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: """The rows sorted by their tag ``k``.""" From f0114fdce93b2f867a55458416cfafb5db72f16d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 18:14:42 -0800 Subject: [PATCH 6/6] fix(pipeline): warn after every failed run; clean up a pod's leftover tasks; classify wrapped async functions (ITL-994, ITL-997) - RowFailuresWarning is issued with a fresh registry at the caller's line, so a second identical failing run (re-running a notebook cell) warns again; the user's filters still apply. - The bridge loop cancels and gathers the tasks still pending before it closes, as asyncio.run does. - DataFunctionWrapper forwards is_async, so an async function behind a result database is recognised in the second-interrupt warning. - The async docs sample sorts the table, whose rows come in completion order. Co-Authored-By: Claude Opus 5.5 --- docs/concepts/function-pods.md | 3 +- src/orcapod/core/data_function.py | 5 ++ src/orcapod/pipeline/async_orchestrator.py | 18 ++++++ src/orcapod/pipeline/job.py | 29 +++++++++- .../test_job_run_in_event_loop.py | 55 +++++++++++++++++-- tests/test_pipeline/test_job_run_summary.py | 24 ++++++++ 6 files changed, 127 insertions(+), 7 deletions(-) diff --git a/docs/concepts/function-pods.md b/docs/concepts/function-pods.md index 8e5c1e5e9..45611d170 100644 --- a/docs/concepts/function-pods.md +++ b/docs/concepts/function-pods.md @@ -418,7 +418,8 @@ with job: job.run(orchestrator=AsyncPipelineOrchestrator()) -print(job.nodes["bmi"].as_table().to_pandas()) +# The async run stores rows as they finish: sort them to print in a fixed order. +print(job.nodes["bmi"].as_table().sort_by("subject_id").to_pandas()) # subject_id bmi # 0 mouse_01 1756.944444 # 1 mouse_02 2210.000000 diff --git a/src/orcapod/core/data_function.py b/src/orcapod/core/data_function.py index f25cd6e0e..b4e47c4c3 100644 --- a/src/orcapod/core/data_function.py +++ b/src/orcapod/core/data_function.py @@ -949,6 +949,11 @@ def __init__(self, data_function: DataFunctionProtocol, **kwargs) -> None: def computed_label(self) -> str | None: return self._data_function.label + @property + def is_async(self) -> bool: + """Whether the wrapped function is an async coroutine function.""" + return getattr(self._data_function, "is_async", False) is True + @property def major_version(self) -> int: return self._data_function.major_version diff --git a/src/orcapod/pipeline/async_orchestrator.py b/src/orcapod/pipeline/async_orchestrator.py index 979ceaab2..07f53b7f7 100644 --- a/src/orcapod/pipeline/async_orchestrator.py +++ b/src/orcapod/pipeline/async_orchestrator.py @@ -384,6 +384,7 @@ def run_loop() -> None: errors.append(exc) finally: try: + _cancel_leftover_tasks(loop) loop.run_until_complete(loop.shutdown_asyncgens()) loop.run_until_complete(loop.shutdown_default_executor()) finally: @@ -419,6 +420,23 @@ def run_loop() -> None: return results[0] +def _cancel_leftover_tasks(loop: asyncio.AbstractEventLoop) -> None: + """Cancel the tasks still pending on ``loop`` and let them finish, as ``asyncio.run`` does. + + A task a pod started and left running would otherwise lose its cancellation + cleanup when the loop closes. + + Args: + loop: The bridge thread's loop, no longer running. + """ + leftover = asyncio.all_tasks(loop) + if not leftover: + return + for task in leftover: + _ = task.cancel() + _ = loop.run_until_complete(asyncio.gather(*leftover, return_exceptions=True)) + + def _wait_until(event: threading.Event) -> None: """Block until ``event`` is set, letting a ``KeyboardInterrupt`` through. diff --git a/src/orcapod/pipeline/job.py b/src/orcapod/pipeline/job.py index f7b26e2db..afe0830b3 100644 --- a/src/orcapod/pipeline/job.py +++ b/src/orcapod/pipeline/job.py @@ -4,6 +4,7 @@ import contextlib import logging +import sys import warnings from collections.abc import Iterator from dataclasses import dataclass @@ -44,6 +45,7 @@ if TYPE_CHECKING: from pathlib import Path + from types import FrameType from orcapod.pipeline.async_orchestrator import AsyncPipelineOrchestrator from orcapod.pipeline.execution_context import ExecutionContext @@ -99,6 +101,29 @@ def _with_counter( return CompositeObserver(observer, counter) # pyright: ignore[reportReturnType] # forwards every hook; its **kwargs signatures do not match the protocol structurally +def _warn_row_failures(summary: RunSummary, caller: FrameType) -> None: + """Issue the run's ``RowFailuresWarning``, shown after every run with failed rows. + + Python's default filter shows a warning once per text and line, so a second identical + failing run from the same line (re-running a notebook cell) would show nothing. A + fresh registry for each run keeps every run's warning, while the user's filters + (``ignore``, ``error``) still apply. + + Args: + summary: The run's summary, with failed rows. + caller: The frame that called ``run``; the warning is shown at its line. + """ + warnings.warn_explicit( + summary.failures_message(), + RowFailuresWarning, + caller.f_code.co_filename, + caller.f_lineno, + module=caller.f_globals.get("__name__"), + registry=None, + module_globals=caller.f_globals, + ) + + class PipelineJob(AbstractPipelineBase[JobNode]): """Pipeline + source bindings + execution context. @@ -930,7 +955,7 @@ def run( ) summary = self._finish_run(prepared, counter) if not summary.ok: - warnings.warn(summary.failures_message(), RowFailuresWarning, stacklevel=2) + _warn_row_failures(summary, sys._getframe(1)) # pyright: ignore[reportPrivateUsage] # the caller's line is where the warning is shown return summary async def run_async( @@ -988,7 +1013,7 @@ async def run_async( ) summary = self._finish_run(prepared, counter) if not summary.ok: - warnings.warn(summary.failures_message(), RowFailuresWarning, stacklevel=2) + _warn_row_failures(summary, sys._getframe(1)) # pyright: ignore[reportPrivateUsage] # the caller's line is where the warning is shown return summary def _prepare_run(self, commit_every_rows: int) -> "_PreparedRun": diff --git a/tests/test_pipeline/test_job_run_in_event_loop.py b/tests/test_pipeline/test_job_run_in_event_loop.py index 264013e0b..8a97d6abc 100644 --- a/tests/test_pipeline/test_job_run_in_event_loop.py +++ b/tests/test_pipeline/test_job_run_in_event_loop.py @@ -361,7 +361,6 @@ async def notebook_cell() -> None: assert 0 in stuck_calls -@function_pod(output_keys="y") async def async_stuck_on_first_row(x: int) -> int: """Row 0 presses the interrupt button twice and blocks the event loop until released.""" if x == 0: @@ -372,19 +371,34 @@ async def async_stuck_on_first_row(x: int) -> int: return x -def test_second_interrupt_names_an_async_function_blocking_the_loop() -> None: +POD_FORMS: dict[str, Callable[[], dict[str, Any]]] = { + "plain": dict, + "result_database": lambda: {"result_database": InMemoryArrowDatabase()}, + "pod_cache_database": lambda: {"pod_cache_database": InMemoryArrowDatabase()}, +} + + +@pytest.mark.parametrize("pod_form", list(POD_FORMS)) +def test_second_interrupt_names_an_async_function_blocking_the_loop( + pod_form: str, +) -> None: """For an async function blocking the loop, the warning says rows may still be stored. The cancellation cannot run on a loop whose thread the function holds, so rows already in progress may be stored once it returns; the warning says so instead of - promising that nothing more is written. The run still stops once it returns. + promising that nothing more is written. The run still stops once it returns. The + function is recognised as async in every form of pod: plain, with a result + database (a wrapped data function) and with a pod cache. """ stuck_release.clear() + pod = function_pod(output_keys="y", **POD_FORMS[pod_form]())( + async_stuck_on_first_row + ) rows = [{"k": f"r{i}", "x": i} for i in range(6)] job = PipelineJob(name="async_stuck_demo", store=InMemoryArrowDatabase()) with job: source = DictSource(data=rows, tag_columns=["k"], source_id="rows") - stuck = async_stuck_on_first_row.pod(source, label="stuck") + stuck = pod.pod(source, label="stuck") after_stuck.pod(stuck, label="after") async def notebook_cell() -> None: @@ -411,3 +425,36 @@ async def notebook_cell() -> None: while bridge_threads() and time.monotonic() < deadline: time.sleep(0.05) assert not bridge_threads() + + +pod_task_cleanup: list[int] = [] + + +async def wait_forever_then_clean_up(x: int) -> None: + """A background task left pending by a pod; records its cancellation cleanup.""" + try: + await asyncio.sleep(3600) + except asyncio.CancelledError: + pod_task_cleanup.append(x) + raise + + +@function_pod(output_keys="y") +async def leaves_a_task_running(x: int) -> int: + """Start a background task and return without awaiting it.""" + _ = asyncio.get_running_loop().create_task(wait_forever_then_clean_up(x)) + return x + + +def test_tasks_left_pending_by_a_pod_are_cancelled_and_cleaned_up() -> None: + """Inside a running loop, as with ``asyncio.run``, leftover tasks get their cleanup.""" + pod_task_cleanup.clear() + rows = [{"k": f"r{i}", "x": i} for i in range(3)] + job = PipelineJob(name="leftover_demo", store=InMemoryArrowDatabase()) + with job: + source = DictSource(data=rows, tag_columns=["k"], source_id="rows") + leaves_a_task_running.pod(source, label="leaves") + + run_inside_running_loop(job) + + assert sorted(pod_task_cleanup) == [0, 1, 2] diff --git a/tests/test_pipeline/test_job_run_summary.py b/tests/test_pipeline/test_job_run_summary.py index ef120feff..9c90124bd 100644 --- a/tests/test_pipeline/test_job_run_summary.py +++ b/tests/test_pipeline/test_job_run_summary.py @@ -248,3 +248,27 @@ def test_traceback_pointer_is_valid_python_for_any_label() -> None: call = ast.parse(pointer, mode="eval").body assert isinstance(call, ast.Call) assert ast.literal_eval(call.args[0]) == label + + +def test_every_failed_run_warns_also_from_the_same_line() -> None: + """Re-running the same failing cell warns again; a user's ignore filter still applies. + + Python's default filter shows a given text from a given line once, and a notebook + names a cell's file by its content, so an identical second run would warn nothing. + """ + # One filter setting for both runs, as in a session: changing the filters between + # the runs would reset Python's once-per-line record and hide the problem. + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("default") + for _ in range(2): + job = make_job(InMemoryArrowDatabase(), always_fails, "broken") + _ = job.run(orchestrator=SyncPipelineOrchestrator()) # the same line twice + assert sum(1 for w in caught if issubclass(w.category, RowFailuresWarning)) == 2 + + job = make_job(InMemoryArrowDatabase(), always_fails, "broken") + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("default") + warnings.filterwarnings("ignore", category=RowFailuresWarning) + summary = job.run(orchestrator=SyncPipelineOrchestrator()) + assert not [w for w in caught if issubclass(w.category, RowFailuresWarning)] + assert summary.failed_rows == 4