From 4cab9ebc71f0e2edafa1678313b2313bc4a28673 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 16:15:05 -0800 Subject: [PATCH 1/9] test(async): failing regression tests for the async fan-out and REPLAY deadlock (ITL-1075) WIP: tests fail by timeout on main; fix follows. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ITL-1075.md | 4 + design_issues/ITL-1075-AH1.md | 24 ++ .../plans/2026-11-13-itl-1075-async-hang.md | 110 ++++++++++ .../test_async_fan_out_does_not_hang.py | 207 ++++++++++++++++++ 4 files changed, 345 insertions(+) create mode 100644 changelog.d/ITL-1075.md create mode 100644 design_issues/ITL-1075-AH1.md create mode 100644 superpowers/plans/2026-11-13-itl-1075-async-hang.md create mode 100644 tests/test_pipeline/test_async_fan_out_does_not_hang.py diff --git a/changelog.d/ITL-1075.md b/changelog.d/ITL-1075.md new file mode 100644 index 000000000..58316a5f5 --- /dev/null +++ b/changelog.d/ITL-1075.md @@ -0,0 +1,4 @@ +### Fixed + +- An async job run no longer hangs when one node feeds both inputs of a `SemiJoin` (ITL-1075). `SemiJoin` read its left input only after its right input had ended. When a node fed both inputs (for example `SemiJoin(C, A)`, with `C` computed from `A`), the bounded channel into the unread left input filled, every node between `A` and that channel blocked in turn, `A` could not finish, and the run waited forever. A computing function node absorbs its whole input, which hid this on a first run: the hang showed on a run that found every row stored, or on any run of a graph of operators only. `SemiJoin` now reads both inputs at once. It keeps the left rows that arrive before the right input ends (at most the whole left input, which the sync path and `MergeJoin` already hold), and then streams the rest of the left input as before. Output rows, their order, system tags and hashes are unchanged. +- An async job run no longer hangs on an operator node in `CacheMode.REPLAY` with more upstream rows than a channel holds (ITL-1075). The node served its stored rows without reading its input, so its upstream blocked on a full channel. It now reads and discards its input while it replays, so the upstream runs over every row, as it does with the sync orchestrator. diff --git a/design_issues/ITL-1075-AH1.md b/design_issues/ITL-1075-AH1.md new file mode 100644 index 000000000..9405316a6 --- /dev/null +++ b/design_issues/ITL-1075-AH1.md @@ -0,0 +1,24 @@ +## `src/orcapod/core/operators/` — Async execution + +### AH1 — An async run deadlocked when a consumer stopped reading one of its inputs +**Status:** in progress +**Severity:** high +**Issue:** ITL-1075 + +The async orchestrator joins nodes with bounded channels, so every node must keep reading +each input channel to its end: a consumer that waits on something before it reads an input +blocks that input's producer once the channel is full. Two consumers did: + +- `SemiJoin.async_execute` collected its right input before it read its left one. When one + node `A` fed both inputs (`SemiJoin(C, A)` with `C` computed from `A`, or `SemiJoin(A, C)`, + or `SemiJoin(A, B)` with `A` and `B` from one source), the channel into the unread left + input filled, every node between `A` and it blocked, `A` could not finish its right-side + rows, and the run waited forever. A computing function node spawns one task per input row + without a limit (no `max_concurrency`) and so absorbs its whole input, which hid the cycle + on a first run; a run that found every row stored, or a graph of operators only, hung once + the rows exceeded what the channels in the cycle hold (about 3 x `buffer_size` + 130). +- `OperatorNode.async_execute` in `CacheMode.REPLAY` served its stored rows without reading + its inputs, so any upstream with more rows than a channel holds blocked forever. + +Every other `async_execute` (the unary operators, `Join`, `MergeJoin` and the barrier +default, function, side-effect and source nodes) reads every input to its end. diff --git a/superpowers/plans/2026-11-13-itl-1075-async-hang.md b/superpowers/plans/2026-11-13-itl-1075-async-hang.md new file mode 100644 index 000000000..d32011167 --- /dev/null +++ b/superpowers/plans/2026-11-13-itl-1075-async-hang.md @@ -0,0 +1,110 @@ +# ITL-1075 Async job run hangs (bounded-channel deadlock) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** An async `PipelineJob.run()` never deadlocks when one node feeds both inputs of a later +`SemiJoin` (or an operator node in `CacheMode.REPLAY` has upstreams), on any run and any `buffer_size`. + +**Architecture:** The deadlock is a consumer that stops reading one input while it waits for +something only that input's producer can unblock. On `main` two consumers do that: +`SemiJoin.async_execute` (collects the right input before reading the left) and +`OperatorNode.async_execute` in `REPLAY` (never reads its inputs). Fix each in place: `SemiJoin` +reads the left input while a task collects the right; a `REPLAY` node drains (discards) its +inputs while it replays. No orchestrator, channel or `buffer_size` change. + +**Tech Stack:** asyncio, pytest (`asyncio.run(asyncio.wait_for(job.run_async(...)))` as the guard). + +Branch: `eywalker/itl-1075-async-job-run-hangs-on-a-re-run-when-one` from `origin/main` `163e5ddc`. + +--- + +## Cause (established, evidence in the coordinator message) + +Task dump of the hung run 2 (1000 rows, buffer 64), each task and the queue it waits on: + +- `SJ` (SemiJoin) `collect` of its right input, `get` on edge A -> SJ (0/64): waits for A to close. +- `A` `route_inputs`, `put` on edge A -> J (64/64): the broadcast writer delivers to J's queue first. +- `J` drain tasks `put` on Join's merged queue (64/64); Join loop `put` on J's intermediate (64/64); + J forward `put` on edge J -> C (64/64). +- `C` `route_inputs`, `put` on edge C -> SJ (64/64): SJ reads its left input only after the right closed. + +Cycle: SJ waits A, A waits J, J waits C, C waits SJ. + +Run 1 passes because a computing function node runs its pod's `async_execute`, which (no +`max_concurrency`) spawns one task per input row with no limit: C absorbs every row J emits, +so J never blocks. Proof: capping `max_concurrency` at 4 makes run 1 hang the same way. On a +cache-hit run `route_inputs` sends hits straight to the output with a blocking send. + +300 rows pass at buffer 64 because the cycle's queues hold about 3 x buffer + 130 rows +(A -> J, J -> C, C -> SJ at `buffer_size`, plus Join's merged queue and the operator node's +intermediate channel at a fixed 64) before anything blocks. + +## Tasks + +### Task 1: regression tests (fail on main by timeout) + +**Files:** Create `tests/test_pipeline/test_async_fan_out_does_not_hang.py` + +- Operators-only graph: `A = MapData(x->a)(src)`, `B = MapData(x->b)(src)`, + `C = MapData(b->c)(DropDataColumns(a)(Join(A, B)))`, `OUT = op(C, A)` for op in + Join / MergeJoin / SemiJoin, and `SemiJoin(A, C)` (fan-out node on the left); + 1000 rows; `buffer_size` 1, 16, 64; runs 1 and 2; each run under + `asyncio.wait_for(job.run_async(...), timeout)`; rows equal a sync job's rows. +- Issue graph with function pods (fa, fb, fc), 300 rows, buffer 1: run 1 and run 2 finish and equal sync. +- REPLAY operator node with 200 rows upstream, buffer 1: run finishes (only if the coordinator + keeps REPLAY in scope). +- [ ] Run, confirm SemiJoin cases (and REPLAY) fail with `TimeoutError` on main. +- [ ] WIP commit. + +### Task 2: SemiJoin reads both inputs at once + +**Files:** Modify `src/orcapod/core/operators/semijoin.py` (`async_execute`) + +```python +async with asyncio.TaskGroup() as tg: + right_task = tg.create_task(right_ch.collect()) + early_left = [] + left_closed = False + while not right_task.done(): + try: + early_left.append(await left_ch.receive()) + except ChannelClosed: + left_closed = True + break +right_rows = right_task.result() + +async def left_rows(): + for row in early_left: + yield row + if not left_closed: + async for row in left_ch: + yield row +``` + +The rest of the method iterates `left_rows()` where it iterated `left_ch`. Output order stays +the left arrival order. Memory: left rows that arrive before the right input closes are held +(at most the whole left input, as the barrier `MergeJoin` holds both inputs and the sync path +holds both tables); after that the left streams with backpressure. + +- [ ] Run Task 1 tests and `tests/test_core/operators/test_async_sync_equivalence.py`, SemiJoin tests. +- [ ] Commit `fix(semijoin): read both inputs at once so a shared upstream cannot deadlock (ITL-1075)`. + +### Task 3 (if approved): REPLAY operator node drains its inputs + +**Files:** Modify `src/orcapod/core/nodes/operator_node.py` (`async_execute`, REPLAY branch): +drain each input channel (`async for _ in ch: pass`) in a task group alongside the replay. + +### Task 4: docs, changelog, design issue + +- `changelog.d/ITL-1075.md` (`### Fixed`). +- `design_issues/ITL-1075-AH1.md` under `## src/orcapod/core/operators/ — Async execution`. +- `uv run python scripts/assemble_changelog.py --check`, `uv run python scripts/design_issues.py --check`. + +### Task 5: verify, PR + +- ruff check/format, basedpyright (exit code), targeted pytest (one process, py3.12), golden tests unchanged. +- Merge origin/main, push, PR `Fixes ITL-1075`, CodeRabbit rounds, CI green on 3.11/3.12/3.13. + +## Progress notes + +- 2026-11-13: cause established (task dump + concurrency cap experiment); shape matrix measured. diff --git a/tests/test_pipeline/test_async_fan_out_does_not_hang.py b/tests/test_pipeline/test_async_fan_out_does_not_hang.py new file mode 100644 index 000000000..47efe7f48 --- /dev/null +++ b/tests/test_pipeline/test_async_fan_out_does_not_hang.py @@ -0,0 +1,207 @@ +"""An async run never deadlocks when one node feeds both inputs of a later operator (ITL-1075). + +The async orchestrator joins nodes with bounded channels. ``SemiJoin`` collected its right +input before it read its left one. When one node fed both inputs, the channel into the +unread left input filled, every node between the shared node and that channel blocked in +turn, the shared node could not finish its right-side rows, and the run waited forever. +A function node that computes absorbs its whole input (one task per row), which hid the +deadlock on a first run; a run that finds every row stored streams with backpressure and +hung. A graph of operators only hangs on its first run. + +An operator node in ``CacheMode.REPLAY`` never read its inputs at all: with more upstream +rows than a channel holds, the upstream blocked and the run hung, with no fan-out. + +Each run here is awaited under ``asyncio.wait_for``: a deadlock fails the test with +``TimeoutError`` instead of hanging the suite. No test sleeps. +""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.core.operators import ( + DropDataColumns, + Join, + MapData, + MergeJoin, + SelectDataColumns, + SemiJoin, +) +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.sources import DictSource +from orcapod.types import CacheMode + +# Seconds one run may take. A run here takes well under two seconds; a deadlocked run +# never ends. +RUN_TIMEOUT = 30.0 + +N_ROWS = 1000 # far above every channel buffer in the cycle + +BUFFER_SIZES = [1, 16, 64] + + +def make_source(n_rows: int) -> DictSource: + """``n_rows`` rows tagged ``(g, k)`` with one data column ``x``.""" + rows = [{"g": i % 4, "k": i, "x": i} for i in range(n_rows)] + return DictSource(rows, tag_columns=["g", "k"], source_id="src") + + +def run_async_job(job: PipelineJob, buffer_size: int) -> None: + """Run ``job`` with the async orchestrator; fail with ``TimeoutError`` on a deadlock.""" + orchestrator = AsyncPipelineOrchestrator(buffer_size=buffer_size) + asyncio.run(asyncio.wait_for(job.run_async(orchestrator=orchestrator), RUN_TIMEOUT)) + + +def output_rows(job: PipelineJob) -> list[dict[str, Any]]: + """The rows of node ``OUT``, with system tags and source info, in a fixed order.""" + table = job.nodes["OUT"].as_table(columns={"system_tags": True, "source": True}) + names = sorted(table.column_names) + return ( + table.select(names).sort_by([(name, "ascending") for name in names]).to_pylist() + ) + + +# --------------------------------------------------------------------------- +# A graph of operators only: A feeds the join that makes C, and the last operator. +# --------------------------------------------------------------------------- + +LAST_OPERATORS = { + "Join(C, A)": lambda c, a: Join()(c, a, label="OUT"), + "MergeJoin(C, A)": lambda c, a: MergeJoin()(c, a, label="OUT"), + "SemiJoin(C, A)": lambda c, a: SemiJoin()(c, a, label="OUT"), + "SemiJoin(A, C)": lambda c, a: SemiJoin()(a, c, label="OUT"), +} + + +def operator_job(last: str) -> PipelineJob: + """``A`` and ``B`` from one source, ``C`` from ``Join(A, B)``, then ``last`` of C and A.""" + job = PipelineJob(name="fan_out", store=InMemoryArrowDatabase()) + with job: + src = make_source(N_ROWS) + a = MapData({"x": "a"})(src, label="A") + b = MapData({"x": "b"})(src, label="B") + joined = Join()(a, b, label="J") + c = MapData({"b": "c"})(DropDataColumns(["a"])(joined, label="D"), label="C") + LAST_OPERATORS[last](c, a) + return job + + +@pytest.mark.parametrize("buffer_size", BUFFER_SIZES) +@pytest.mark.parametrize("last", list(LAST_OPERATORS)) +def test_operator_fan_out_completes_on_both_runs(last: str, buffer_size: int) -> None: + """Runs 1 and 2 finish and give the rows of a sync run.""" + expected_job = operator_job(last) + expected_job.run(orchestrator=SyncPipelineOrchestrator()) + expected = output_rows(expected_job) + assert expected # the comparison is not vacuous + + job = operator_job(last) + for _ in range(2): + run_async_job(job, buffer_size) + assert output_rows(job) == expected + + +# --------------------------------------------------------------------------- +# The graph of the issue: function pods A, B, C; SemiJoin(C, A). +# --------------------------------------------------------------------------- + + +@function_pod(output_keys="a") +def plus_one(x: int) -> int: + """Return ``x + 1``.""" + return x + 1 + + +@function_pod(output_keys="b") +def plus_two(x: int) -> int: + """Return ``x + 2``.""" + return x + 2 + + +@function_pod(output_keys="c") +def add(a: int, b: int) -> int: + """Return ``a + b``.""" + return a + b + + +def function_job(store: InMemoryArrowDatabase, n_rows: int) -> PipelineJob: + """The issue's graph: ``SemiJoin(C, A)`` where ``A`` also feeds ``C``.""" + job = PipelineJob(name="fan_out_pods", store=store) + with job: + src = make_source(n_rows) + a = plus_one.pod(src, label="A") + b = plus_two.pod(src, label="B") + c = add.pod(Join()(a, b, label="J"), label="C") + SemiJoin()(c, a, label="OUT") + return job + + +def test_function_fan_out_completes_when_every_row_is_stored() -> None: + """The second run, which finds every row stored, finishes with the rows of the first. + + 300 rows at buffer size 1 deadlocked on ``main``; 1,000 rows at 1, 16 and 64 are in + the measured table of the pull request (too slow for the default suite). + """ + expected_job = function_job(InMemoryArrowDatabase(), 300) + expected_job.run(orchestrator=SyncPipelineOrchestrator()) + expected = expected_job.nodes["OUT"].as_table() + + job = function_job(InMemoryArrowDatabase(), 300) + for _ in range(2): + run_async_job(job, buffer_size=1) + got = job.nodes["OUT"].as_table() + assert got.sort_by([("k", "ascending")]).equals( + expected.sort_by([("k", "ascending")]) + ) + + +# --------------------------------------------------------------------------- +# An operator node in REPLAY mode: it serves its stored rows and must still read its input. +# --------------------------------------------------------------------------- + +REPLAY_ROWS = 200 # above both buffer sizes tested + + +def replay_job(store: InMemoryArrowDatabase) -> PipelineJob: + """``A`` (a function pod) feeding operator node ``OUT``, which replays stored rows. + + ``OUT`` first runs once in ``CacheMode.LOG`` with the sync orchestrator, so it has + stored rows; then it is switched to ``CacheMode.REPLAY``. + """ + job = PipelineJob(name="replay", store=store) + with job: + a = plus_one.pod(make_source(REPLAY_ROWS), label="A") + SelectDataColumns(["a"])(a, label="OUT") + out = job.nodes["OUT"] + # attach_databases is the public way to turn on LOG / REPLAY mode. + out.attach_databases(pipeline_database=store.at("replay"), cache_mode=CacheMode.LOG) + job.run(orchestrator=SyncPipelineOrchestrator()) + out.attach_databases( + pipeline_database=store.at("replay"), cache_mode=CacheMode.REPLAY + ) + return job + + +@pytest.mark.parametrize("buffer_size", [1, 64]) +def test_replay_operator_node_reads_its_input(buffer_size: int) -> None: + """A REPLAY run finishes and does what a sync REPLAY run does: no more, no less. + + In both, the upstream ``A`` still runs over every row (the sync orchestrator executes + every node), and ``OUT`` gives its stored rows. + """ + expected_job = replay_job(InMemoryArrowDatabase()) + expected_job.run(orchestrator=SyncPipelineOrchestrator()) + + job = replay_job(InMemoryArrowDatabase()) + run_async_job(job, buffer_size) + + for label in ("A", "OUT"): + got = job.nodes[label].as_table().sort_by([("k", "ascending")]) + expected = expected_job.nodes[label].as_table().sort_by([("k", "ascending")]) + assert got.num_rows == REPLAY_ROWS + assert got.equals(expected) From f68e6e345e56789765bd197005af463b756eebb0 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 16:23:50 -0800 Subject: [PATCH 2/9] fix(async): SemiJoin reads both inputs at once; a REPLAY operator node drains its inputs (ITL-1075) SemiJoin collected its right input before reading its left one, so a node feeding both inputs deadlocked the async run once the channels in the cycle were full. It now reads the left input while a task collects the right one. An operator node in CacheMode.REPLAY never read its inputs; it now reads and discards them while it replays, as the sync orchestrator runs the upstream. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/operator_node.py | 31 +++++++-- src/orcapod/core/operators/semijoin.py | 88 +++++++++++++++++++++---- 2 files changed, 101 insertions(+), 18 deletions(-) diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index c7c357f3d..8effef7cb 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -79,6 +79,16 @@ pc = LazyModule("pyarrow.compute") +async def _discard(channel: ReadableChannel[Any]) -> None: + """Read ``channel`` to its end, keeping nothing. + + Args: + channel: The channel to drain. + """ + async for _ in channel: + pass + + # --------------------------------------------------------------------------- # OperatorNodeBase — shared base (no DB) # --------------------------------------------------------------------------- @@ -1659,7 +1669,8 @@ async def async_execute( Without a database, or with one in OFF mode, nothing is stored. With a database: - - REPLAY: emit from DB, close output. + - REPLAY: emit from DB while reading and discarding every input, so + the upstream nodes run as with the sync orchestrator; close output. - OFF: delegate to operator, forward and keep results. - LOG: delegate to operator, forward and keep results, then store them in the DB. @@ -1687,10 +1698,20 @@ async def async_execute( self._pipeline_database is not None and self._cache_mode == CacheMode.REPLAY ): - self._replay_from_cache() - assert self._cached_output_stream is not None - for tag, data in self._cached_output_stream.iter_data(): - await output.send((tag, data)) + # Read and discard every input while the stored rows are sent: the + # upstream nodes run over every row, as with the sync orchestrator, + # and an input no one reads blocks its producer, and the run, once + # its channel is full (ITL-1075). + discards = [asyncio.create_task(_discard(ch)) for ch in inputs] + try: + self._replay_from_cache() + assert self._cached_output_stream is not None + for tag, data in self._cached_output_stream.iter_data(): + await output.send((tag, data)) + await asyncio.gather(*discards) + finally: + for discard in discards: + discard.cancel() ctx_obs.on_node_end(node_label, node_hash) return # finally block closes output diff --git a/src/orcapod/core/operators/semijoin.py b/src/orcapod/core/operators/semijoin.py index fe1929cec..f9b13540f 100644 --- a/src/orcapod/core/operators/semijoin.py +++ b/src/orcapod/core/operators/semijoin.py @@ -1,7 +1,10 @@ -from collections.abc import Sequence +import asyncio +import contextlib +from collections import deque +from collections.abc import AsyncIterator, Sequence from typing import TYPE_CHECKING, Any -from orcapod.channels import ReadableChannel, WritableChannel +from orcapod.channels import ChannelClosed, ReadableChannel, WritableChannel from orcapod.core.operators.base import BinaryOperator, require_shared_arrow_type from orcapod.core.operators.match_keys import frozen_scalar from orcapod.core.streams import ArrowTableStream @@ -174,9 +177,16 @@ async def async_execute( """Build-probe: collect right input, then stream left through a hash lookup. Phase 1 — Build: collect all rows from the right (filter) channel and - index them by the common-key values. - Phase 2 — Probe: stream left rows one at a time; for each row whose - common-key values appear in the right-side index, emit immediately. + index them by the common-key values. The left channel is read at the + same time (ITL-1075): the left rows that arrive before the right input + ends are kept, at most the whole left input, as the sync path and + ``MergeJoin``'s barrier path hold their inputs. Reading the left input + only after the right one ended deadlocked a run in which one node fed + both inputs. + Phase 2 — Probe: the kept left rows, then the rest of the left input + one at a time with backpressure; for each row whose common-key values + appear in the right-side index, emit immediately. Left rows keep their + arrival order. Falls back to barrier mode when the right input is empty (schema cannot be inferred from data) or when there are no common keys. @@ -194,10 +204,10 @@ async def async_execute( input's pipeline hash. """ try: - left_ch, right_ch = inputs[0], inputs[1] - - # Phase 1: Build right-side lookup - right_rows = await right_ch.collect() + # Phase 1: Build right-side lookup, reading the left input meanwhile + right_rows, left_rows = await _collect_right_reading_left( + inputs[0], inputs[1] + ) if not right_rows: # Empty right: determine common keys from the validated @@ -206,10 +216,11 @@ async def async_execute( common = self._common_keys_from_schema() if common: # Drain left channel (discard) — result is empty - await left_ch.collect() + async for _ in left_rows: + pass return # No common keys — pass all left rows through unchanged - async for tag, data in left_ch: + async for tag, data in left_rows: await output.send((tag, data)) return @@ -222,7 +233,7 @@ async def async_execute( common_keys: tuple[str, ...] | None = None right_lookup: set[tuple[object, ...]] = set() - async for tag, data in left_ch: + async for tag, data in left_rows: if common_keys is None: # First left row — determine common keys and build index left_tag_keys = set(tag.keys()) @@ -233,7 +244,7 @@ async def async_execute( if not common_keys: # No common keys — pass all left rows through await output.send((tag, data)) - async for t, p in left_ch: + async for t, p in left_rows: await output.send((t, p)) return @@ -305,3 +316,54 @@ def _match_key( def identity_structure(self) -> Any: return self.__class__.__name__ + + +async def _collect_right_reading_left( + left: ReadableChannel[tuple[TagProtocol, DataProtocol]], + right: ReadableChannel[tuple[TagProtocol, DataProtocol]], +) -> tuple[ + list[tuple[TagProtocol, DataProtocol]], + AsyncIterator[tuple[TagProtocol, DataProtocol]], +]: + """Collect ``right`` while reading ``left``, so neither input stops being read. + + A task collects ``right``; meanwhile the rows of ``left`` are read and kept, until + ``right`` ends or ``left`` does. A semi-join that waited for its right input before + reading its left one blocked a node feeding both inputs: the channel into the left + input filled, that node could not finish its right-side rows, and the run waited + forever (ITL-1075). + + Args: + left: The left input channel. + right: The right input channel. + + Returns: + The rows of ``right``, and an iterator over every row of ``left`` in arrival + order: the rows kept while ``right`` was collected, then the rest of ``left`` + as it arrives. + """ + right_task = asyncio.create_task(right.collect()) + kept: deque[tuple[TagProtocol, DataProtocol]] = deque() + left_ended = False + try: + while not right_task.done(): + try: + kept.append(await left.receive()) + except ChannelClosed: + left_ended = True + break + right_rows = await right_task + finally: + if not right_task.done(): + right_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await right_task + + async def left_rows() -> AsyncIterator[tuple[TagProtocol, DataProtocol]]: + while kept: + yield kept.popleft() + if not left_ended: + async for row in left: + yield row + + return right_rows, left_rows() From fe925878220fe62ff905f07f1a2bcca972d0e216 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 16:26:28 -0800 Subject: [PATCH 3/9] test(async): compute each sync expectation once (ITL-1075) Co-Authored-By: Claude Opus 5.5 --- .../test_async_fan_out_does_not_hang.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/test_pipeline/test_async_fan_out_does_not_hang.py b/tests/test_pipeline/test_async_fan_out_does_not_hang.py index 47efe7f48..db1614ac5 100644 --- a/tests/test_pipeline/test_async_fan_out_does_not_hang.py +++ b/tests/test_pipeline/test_async_fan_out_does_not_hang.py @@ -18,6 +18,7 @@ from __future__ import annotations import asyncio +import functools from typing import Any import pytest @@ -91,13 +92,19 @@ def operator_job(last: str) -> PipelineJob: return job +@functools.cache +def sync_operator_rows(last: str) -> list[dict[str, Any]]: + """The rows of ``operator_job(last)`` run once with the sync orchestrator.""" + job = operator_job(last) + job.run(orchestrator=SyncPipelineOrchestrator()) + return output_rows(job) + + @pytest.mark.parametrize("buffer_size", BUFFER_SIZES) @pytest.mark.parametrize("last", list(LAST_OPERATORS)) def test_operator_fan_out_completes_on_both_runs(last: str, buffer_size: int) -> None: """Runs 1 and 2 finish and give the rows of a sync run.""" - expected_job = operator_job(last) - expected_job.run(orchestrator=SyncPipelineOrchestrator()) - expected = output_rows(expected_job) + expected = sync_operator_rows(last) assert expected # the comparison is not vacuous job = operator_job(last) From f8dd316383b25d621aa1bead3b8e6c5cf9f9e4e5 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 16:31:37 -0800 Subject: [PATCH 4/9] docs(design_issues): AH1 resolved (ITL-1075) Co-Authored-By: Claude Opus 5.5 --- design_issues/ITL-1075-AH1.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/design_issues/ITL-1075-AH1.md b/design_issues/ITL-1075-AH1.md index 9405316a6..5094a4eb9 100644 --- a/design_issues/ITL-1075-AH1.md +++ b/design_issues/ITL-1075-AH1.md @@ -1,7 +1,7 @@ ## `src/orcapod/core/operators/` — Async execution ### AH1 — An async run deadlocked when a consumer stopped reading one of its inputs -**Status:** in progress +**Status:** resolved **Severity:** high **Issue:** ITL-1075 @@ -22,3 +22,9 @@ blocks that input's producer once the channel is full. Two consumers did: Every other `async_execute` (the unary operators, `Join`, `MergeJoin` and the barrier default, function, side-effect and source nodes) reads every input to its end. + +**Fix:** `SemiJoin` reads its left input while a task collects its right one +(`_collect_right_reading_left`), keeping the left rows that arrive before the right input +ends (at most the whole left input, as the sync path holds); a `REPLAY` operator node +reads and discards its inputs while it sends its stored rows. Regression tests in +`tests/test_pipeline/test_async_fan_out_does_not_hang.py`. From a63d84eaeac41227d552d8092b244e4a4b85031d Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 18:12:55 -0800 Subject: [PATCH 5/9] fix(semijoin): act on the end of the right input at once (ITL-1075) The left receive now runs as a task awaited together with the right collection, so a finished or failed right input is acted on at once, not when the left input next sends or closes. A receive still pending then is handed to the left rows and awaited in its place, never cancelled while its row may be read; left arrival order is kept. On a failure the pending receive is cancelled and awaited. Event-controlled tests: a kept matching row is emitted while the left input is open, a right-side error surfaces while it is open, and the row the pending receive gets is emitted once, in order. Co-Authored-By: Claude Opus 5.5 --- design_issues/ITL-1075-AH1.md | 7 +- src/orcapod/core/operators/semijoin.py | 128 +++++++++++++---- .../plans/2026-11-13-itl-1075-async-hang.md | 27 ++++ .../test_native_async_operators.py | 131 +++++++++++++++++- 4 files changed, 266 insertions(+), 27 deletions(-) diff --git a/design_issues/ITL-1075-AH1.md b/design_issues/ITL-1075-AH1.md index 5094a4eb9..9d873ad61 100644 --- a/design_issues/ITL-1075-AH1.md +++ b/design_issues/ITL-1075-AH1.md @@ -26,5 +26,8 @@ default, function, side-effect and source nodes) reads every input to its end. **Fix:** `SemiJoin` reads its left input while a task collects its right one (`_collect_right_reading_left`), keeping the left rows that arrive before the right input ends (at most the whole left input, as the sync path holds); a `REPLAY` operator node -reads and discards its inputs while it sends its stored rows. Regression tests in -`tests/test_pipeline/test_async_fan_out_does_not_hang.py`. +reads and discards its inputs while it sends its stored rows. Each left receive is +awaited together with the right task, so the end or failure of the right input is acted on +at once; a receive still pending then is handed to the left rows, never cancelled while its +row may be read. Regression tests in `tests/test_pipeline/test_async_fan_out_does_not_hang.py` +and `tests/test_channels/test_native_async_operators.py`. diff --git a/src/orcapod/core/operators/semijoin.py b/src/orcapod/core/operators/semijoin.py index f9b13540f..76f1a5d9f 100644 --- a/src/orcapod/core/operators/semijoin.py +++ b/src/orcapod/core/operators/semijoin.py @@ -1,8 +1,8 @@ import asyncio import contextlib from collections import deque -from collections.abc import AsyncIterator, Sequence -from typing import TYPE_CHECKING, Any +from collections.abc import AsyncGenerator, AsyncIterator, Sequence +from typing import TYPE_CHECKING, Any, final from orcapod.channels import ChannelClosed, ReadableChannel, WritableChannel from orcapod.core.operators.base import BinaryOperator, require_shared_arrow_type @@ -203,6 +203,7 @@ async def async_execute( their system-tag names, so no column is named after an input's pipeline hash. """ + left_rows: _LeftRows | None = None try: # Phase 1: Build right-side lookup, reading the left input meanwhile right_rows, left_rows = await _collect_right_reading_left( @@ -269,6 +270,8 @@ async def async_execute( await output.send((tag, data)) finally: await output.close() + if left_rows is not None: + await left_rows.aclose() @staticmethod def _key_table(tag: TagProtocol, data: DataProtocol, key: str) -> "pa.Table": @@ -318,13 +321,13 @@ def identity_structure(self) -> Any: return self.__class__.__name__ +_Row = tuple[TagProtocol, DataProtocol] + + async def _collect_right_reading_left( - left: ReadableChannel[tuple[TagProtocol, DataProtocol]], - right: ReadableChannel[tuple[TagProtocol, DataProtocol]], -) -> tuple[ - list[tuple[TagProtocol, DataProtocol]], - AsyncIterator[tuple[TagProtocol, DataProtocol]], -]: + left: ReadableChannel[_Row], + right: ReadableChannel[_Row], +) -> tuple[list[_Row], "_LeftRows"]: """Collect ``right`` while reading ``left``, so neither input stops being read. A task collects ``right``; meanwhile the rows of ``left`` are read and kept, until @@ -333,37 +336,114 @@ async def _collect_right_reading_left( input filled, that node could not finish its right-side rows, and the run waited forever (ITL-1075). + Each left receive runs as a task, awaited together with the right one, so the end + (or failure) of ``right`` is acted on at once, not when ``left`` next sends or + closes. A receive still pending then is handed to the returned rows, which await + it in its place: it is never cancelled while its row may still be read. + Args: left: The left input channel. right: The right input channel. Returns: - The rows of ``right``, and an iterator over every row of ``left`` in arrival - order: the rows kept while ``right`` was collected, then the rest of ``left`` - as it arrives. + The rows of ``right``, and every row of ``left`` in arrival order: the rows + kept while ``right`` was collected, the pending receive's row, then the rest + of ``left`` as it arrives. + + Raises: + Exception: Whatever collecting ``right`` raises. A pending left receive is + then cancelled, since the left rows are not going to be read. """ right_task = asyncio.create_task(right.collect()) - kept: deque[tuple[TagProtocol, DataProtocol]] = deque() + kept: deque[_Row] = deque() + receive: asyncio.Task[_Row] | None = None left_ended = False try: while not right_task.done(): - try: - kept.append(await left.receive()) - except ChannelClosed: - left_ended = True - break + if receive is None: + receive = asyncio.create_task(left.receive()) + _ = await asyncio.wait( + (receive, right_task), return_when=asyncio.FIRST_COMPLETED + ) + if receive.done(): + finished, receive = receive, None + try: + kept.append(finished.result()) + except ChannelClosed: + left_ended = True + break right_rows = await right_task + except BaseException: + if receive is not None: + await _cancel_receive(receive) + raise finally: if not right_task.done(): - right_task.cancel() + _ = right_task.cancel() with contextlib.suppress(asyncio.CancelledError): await right_task + return right_rows, _LeftRows(left, kept, receive, left_ended) + + +async def _cancel_receive(receive: "asyncio.Task[_Row]") -> None: + """Cancel a left receive whose row is not going to be read, and wait for it to end. + + Args: + receive: The receive task; it may already be done. + """ + _ = receive.cancel() + _ = await asyncio.wait((receive,)) + if not receive.cancelled(): + # Retrieve its outcome (a row, or ``ChannelClosed``) so it is not reported. + _ = receive.exception() - async def left_rows() -> AsyncIterator[tuple[TagProtocol, DataProtocol]]: - while kept: - yield kept.popleft() - if not left_ended: - async for row in left: + +@final +class _LeftRows: + """Every row of the left input in arrival order, for one pass. + + The rows kept while the right input was collected come first, then the row of + the receive that was pending when it ended, then the rest of the left channel. + """ + + def __init__( + self, + left: ReadableChannel[_Row], + kept: deque[_Row], + pending: "asyncio.Task[_Row] | None", + left_ended: bool, + ) -> None: + self._left = left + self._kept = kept + self._pending = pending + self._left_ended = left_ended + self._rows = self._iterate() + + def __aiter__(self) -> AsyncIterator[_Row]: + return self._rows + + async def _iterate(self) -> AsyncGenerator[_Row, None]: + while self._kept: + yield self._kept.popleft() + if self._pending is not None: + try: + row = await self._pending + except ChannelClosed: + return + finally: + self._pending = None + yield row + if not self._left_ended: + async for row in self._left: yield row - return right_rows, left_rows() + async def aclose(self) -> None: + """Release the left input when its rows are not all read. + + A pending receive is cancelled. After a complete pass there is none, so this + does nothing. + """ + await self._rows.aclose() + if self._pending is not None: + pending, self._pending = self._pending, None + await _cancel_receive(pending) diff --git a/superpowers/plans/2026-11-13-itl-1075-async-hang.md b/superpowers/plans/2026-11-13-itl-1075-async-hang.md index d32011167..769d9b8d4 100644 --- a/superpowers/plans/2026-11-13-itl-1075-async-hang.md +++ b/superpowers/plans/2026-11-13-itl-1075-async-hang.md @@ -108,3 +108,30 @@ drain each input channel (`async for _ in ch: pass`) in a task group alongside t ## Progress notes - 2026-11-13: cause established (task dump + concurrency cap experiment); shape matrix measured. + +## CodeRabbit review round (review 5900648222 of `f8dd3163`, 2026-11-14) + +Findings: one inline Major (`semijoin.py` ~351), and two LanguageTool notes in the review body. + +- Major: `_collect_right_reading_left` checks `right_task.done()` only before awaiting + `left.receive()`. Reproduced (scratch script): with one matching left row sent and the left + channel open, right closed, SemiJoin emits nothing until the left sends again or closes; a + `right.collect()` that raises surfaces only after the left closes. No pipeline deadlock is + reachable with today's consumers (every consumer reads all of its inputs at once), so the effect + is delayed probing and a delayed right error. Fix: + - wait on the pending left receive (a task) and `right_task` together + (`asyncio.wait(..., FIRST_COMPLETED)`); a finished receive's row is kept (or `ChannelClosed` + ends the left); when `right_task` is done, stop; + - never cancel the pending receive on the success path: the left iterator yields the kept rows, + then awaits the pending receive, then reads the rest, so the left arrival order is kept; + - on a failure (right raises, cancellation, a probe error) the pending receive is cancelled: + the run is failing and the left rows are discarded anyway; + - tests (event-controlled, `asyncio.wait_for` only as a hang guard, no sleeps) in + `tests/test_channels/test_native_async_operators.py`: matching row emitted while left open; + right-side error raised while left open. +- LanguageTool "that" -> "who" (design issue line 9, plan line 8): "consumer" is a coroutine, + not a person; decline. +- 2026-11-14: coordinator approved the round plan, plus a test that a left row received by the + pending receive when the right ends is neither lost nor duplicated. Three tests in + `TestSemiJoinRightEndsWhileLeftOpen` failed with `TimeoutError` on `f8dd3163`; pass after the + fix (`_LeftRows` holds the pending receive); equivalence and the 15 fan-out cases rerun green. diff --git a/tests/test_channels/test_native_async_operators.py b/tests/test_channels/test_native_async_operators.py index 68d681681..b4cdaf0c3 100644 --- a/tests/test_channels/test_native_async_operators.py +++ b/tests/test_channels/test_native_async_operators.py @@ -40,7 +40,7 @@ SemiJoin, ) from orcapod.core.streams.arrow_table_stream import ArrowTableStream -from orcapod.protocols.core_protocols import StreamProtocol +from orcapod.protocols.core_protocols import DataProtocol, StreamProtocol, TagProtocol from orcapod.system_constants import constants from orcapod.types import ContentHash @@ -1041,6 +1041,135 @@ async def test_large_input_streaming(self): assert result_ids == expected_ids +# =================================================================== +# SemiJoin — the right input ends while the left one is still open (ITL-1075) +# =================================================================== + + +def _id_stream(ids: list[int], column: str) -> ArrowTableStream: + """Stream with an ``id`` tag and one int64 data column named ``column``.""" + table = pa.table( + { + "id": pa.array(ids, type=pa.int64()), + column: pa.array([i * 10 for i in ids], type=pa.int64()), + } + ) + return ArrowTableStream(table, tag_columns=["id"]) + + +_Row = tuple[TagProtocol, DataProtocol] + + +class _FailingChannel: + """A readable channel whose reading fails: it stands for a failed right input.""" + + async def receive(self) -> _Row: + raise RuntimeError("right input failed") + + def __aiter__(self) -> "_FailingChannel": + return self + + async def __anext__(self) -> _Row: + return await self.receive() + + async def collect(self) -> list[_Row]: + return [await self.receive()] + + +# Guards a wait that must end at once; a timeout means the operator hung. +_HANG_GUARD_SECONDS = 10 + + +class TestSemiJoinRightEndsWhileLeftOpen: + """The end of the right input is acted on at once, not at the next left row. + + Every step waits on a channel event; ``asyncio.wait_for`` only guards against a hang. + """ + + @pytest.mark.asyncio + async def test_kept_matching_row_is_emitted_while_left_is_open(self): + left = _id_stream([2, 7], "a") + right = _id_stream([2, 3], "b") + left_rows = list(left.iter_data()) + left_ch, right_ch, output_ch = ( + Channel[_Row](4), + Channel[_Row](4), + Channel[_Row](4), + ) + await left_ch.writer.send(left_rows[0]) + await feed(right, right_ch) + + task = asyncio.create_task( + SemiJoin().async_execute( + [left_ch.reader, right_ch.reader], + output_ch.writer, + input_pipeline_hashes=hashes(left, right), + ) + ) + tag, _ = await asyncio.wait_for(output_ch.reader.receive(), _HANG_GUARD_SECONDS) + assert tag.as_dict()["id"] == 2 + assert not task.done() # the left input is still open + + await left_ch.writer.send(left_rows[1]) + await left_ch.writer.close() + await asyncio.wait_for(task, _HANG_GUARD_SECONDS) + assert await output_ch.reader.collect() == [] + + @pytest.mark.asyncio + async def test_right_input_error_surfaces_while_left_is_open(self): + left = _id_stream([2], "a") + left_ch, output_ch = Channel[_Row](4), Channel[_Row](4) + await left_ch.writer.send(next(iter(left.iter_data()))) + + with pytest.raises(RuntimeError, match="right input failed"): + await asyncio.wait_for( + SemiJoin().async_execute( + [left_ch.reader, _FailingChannel()], + output_ch.writer, + input_pipeline_hashes=hashes(left, left), + ), + _HANG_GUARD_SECONDS, + ) + assert await output_ch.reader.collect() == [] # the output is closed + # Nothing is left reading the left input. + assert asyncio.all_tasks() == {asyncio.current_task()} + + @pytest.mark.asyncio + async def test_left_row_in_flight_when_right_ends_is_kept_once_in_order(self): + # When the right input ends, a receive on the empty left channel is pending. + # The row it then gets is emitted once, after the kept row and before the + # rows that follow it. + left = _id_stream([2, 3, 9, 4], "a") + right = _id_stream([2, 3, 4], "b") + left_rows = list(left.iter_data()) + left_ch, right_ch, output_ch = ( + Channel[_Row](4), + Channel[_Row](4), + Channel[_Row](4), + ) + await left_ch.writer.send(left_rows[0]) + await feed(right, right_ch) + + task = asyncio.create_task( + SemiJoin().async_execute( + [left_ch.reader, right_ch.reader], + output_ch.writer, + input_pipeline_hashes=hashes(left, right), + ) + ) + first, _ = await asyncio.wait_for( + output_ch.reader.receive(), _HANG_GUARD_SECONDS + ) + for row in left_rows[1:]: + await left_ch.writer.send(row) + await left_ch.writer.close() + await asyncio.wait_for(task, _HANG_GUARD_SECONDS) + + rest = await output_ch.reader.collect() + ids = [first.as_dict()["id"]] + [tag.as_dict()["id"] for tag, _ in rest] + assert ids == [2, 3, 4] + + # =================================================================== # Join — native async # =================================================================== From f90ab64fc0fec4789d0fdf36b295ef6a8f26b10e Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 18:23:14 -0800 Subject: [PATCH 6/9] test(semijoin): make the hand-over test hold a pending left receive (ITL-1075) The right input was closed before SemiJoin started, so the right collection and the first left receive finished in one step and no receive was pending to hand over. The right input now stays open until the left reader signals that its second receive is waiting on an empty channel; dropping the handed-over row makes the test fail. Co-Authored-By: Claude Opus 5.5 --- .../plans/2026-11-13-itl-1075-async-hang.md | 4 ++ .../test_native_async_operators.py | 45 ++++++++++++++++--- 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/superpowers/plans/2026-11-13-itl-1075-async-hang.md b/superpowers/plans/2026-11-13-itl-1075-async-hang.md index 769d9b8d4..7206bb2ff 100644 --- a/superpowers/plans/2026-11-13-itl-1075-async-hang.md +++ b/superpowers/plans/2026-11-13-itl-1075-async-hang.md @@ -135,3 +135,7 @@ Findings: one inline Major (`semijoin.py` ~351), and two LanguageTool notes in t pending receive when the right ends is neither lost nor duplicated. Three tests in `TestSemiJoinRightEndsWhileLeftOpen` failed with `TimeoutError` on `f8dd3163`; pass after the fix (`_LeftRows` holds the pending receive); equivalence and the 15 fan-out cases rerun green. +- 2026-11-14: CodeRabbit review 5900799249 of `a63d84ea` (one body-only Trivial nitpick): the + hand-over test never had a pending receive (right input closed before the start; instrumented: + `pending=False`). Rewritten: the left reader signals its second receive, the right input closes + only then (`pending=True`); dropping the hand-over row makes it fail (`[2, 4] != [2, 3, 4]`). diff --git a/tests/test_channels/test_native_async_operators.py b/tests/test_channels/test_native_async_operators.py index b4cdaf0c3..2ed3b3827 100644 --- a/tests/test_channels/test_native_async_operators.py +++ b/tests/test_channels/test_native_async_operators.py @@ -27,7 +27,7 @@ import pyarrow as pa import pytest -from orcapod.channels import Channel +from orcapod.channels import Channel, ChannelClosed from orcapod.core.operators import ( Batch, DropDataColumns, @@ -1076,6 +1076,34 @@ async def collect(self) -> list[_Row]: return [await self.receive()] +class _SignallingReader: + """Reads a channel and sets ``waiting`` when the given receive call starts.""" + + def __init__(self, channel: Channel[_Row], signal_on_call: int) -> None: + self._reader = channel.reader + self._signal_on_call = signal_on_call + self._calls = 0 + self.waiting = asyncio.Event() + + async def receive(self) -> _Row: + self._calls += 1 + if self._calls == self._signal_on_call: + self.waiting.set() + return await self._reader.receive() + + def __aiter__(self) -> "_SignallingReader": + return self + + async def __anext__(self) -> _Row: + try: + return await self.receive() + except ChannelClosed: + raise StopAsyncIteration + + async def collect(self) -> list[_Row]: + return [row async for row in self] + + # Guards a wait that must end at once; a timeout means the operator hung. _HANG_GUARD_SECONDS = 10 @@ -1136,9 +1164,9 @@ async def test_right_input_error_surfaces_while_left_is_open(self): @pytest.mark.asyncio async def test_left_row_in_flight_when_right_ends_is_kept_once_in_order(self): - # When the right input ends, a receive on the empty left channel is pending. - # The row it then gets is emitted once, after the kept row and before the - # rows that follow it. + # The right input ends while a receive on the empty left channel is pending. + # The row that receive then gets is emitted once, after the kept row and + # before the rows that follow it. left = _id_stream([2, 3, 9, 4], "a") right = _id_stream([2, 3, 4], "b") left_rows = list(left.iter_data()) @@ -1147,16 +1175,21 @@ async def test_left_row_in_flight_when_right_ends_is_kept_once_in_order(self): Channel[_Row](4), Channel[_Row](4), ) + left_reader = _SignallingReader(left_ch, signal_on_call=2) await left_ch.writer.send(left_rows[0]) - await feed(right, right_ch) + for row in right.iter_data(): + await right_ch.writer.send(row) # the right input stays open for now task = asyncio.create_task( SemiJoin().async_execute( - [left_ch.reader, right_ch.reader], + [left_reader, right_ch.reader], output_ch.writer, input_pipeline_hashes=hashes(left, right), ) ) + # The first left row is kept; the second receive waits on an empty channel. + _ = await asyncio.wait_for(left_reader.waiting.wait(), _HANG_GUARD_SECONDS) + await right_ch.writer.close() first, _ = await asyncio.wait_for( output_ch.reader.receive(), _HANG_GUARD_SECONDS ) From 66c3e6c1882fefe1dc9c10023ced272cab323a56 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 18:45:55 -0800 Subject: [PATCH 7/9] test(async): PipelineJob takes database= after the merge of main (ITL-1075) main renamed PipelineJob(store=...) to PipelineJob(database=...) (ITL-1082); the fan-out regression tests now pass database=. Co-Authored-By: Claude Opus 5.5 --- tests/test_pipeline/test_async_fan_out_does_not_hang.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_pipeline/test_async_fan_out_does_not_hang.py b/tests/test_pipeline/test_async_fan_out_does_not_hang.py index db1614ac5..b00e2c5c3 100644 --- a/tests/test_pipeline/test_async_fan_out_does_not_hang.py +++ b/tests/test_pipeline/test_async_fan_out_does_not_hang.py @@ -81,7 +81,7 @@ def output_rows(job: PipelineJob) -> list[dict[str, Any]]: def operator_job(last: str) -> PipelineJob: """``A`` and ``B`` from one source, ``C`` from ``Join(A, B)``, then ``last`` of C and A.""" - job = PipelineJob(name="fan_out", store=InMemoryArrowDatabase()) + job = PipelineJob(name="fan_out", database=InMemoryArrowDatabase()) with job: src = make_source(N_ROWS) a = MapData({"x": "a"})(src, label="A") @@ -138,7 +138,7 @@ def add(a: int, b: int) -> int: def function_job(store: InMemoryArrowDatabase, n_rows: int) -> PipelineJob: """The issue's graph: ``SemiJoin(C, A)`` where ``A`` also feeds ``C``.""" - job = PipelineJob(name="fan_out_pods", store=store) + job = PipelineJob(name="fan_out_pods", database=store) with job: src = make_source(n_rows) a = plus_one.pod(src, label="A") @@ -180,7 +180,7 @@ def replay_job(store: InMemoryArrowDatabase) -> PipelineJob: ``OUT`` first runs once in ``CacheMode.LOG`` with the sync orchestrator, so it has stored rows; then it is switched to ``CacheMode.REPLAY``. """ - job = PipelineJob(name="replay", store=store) + job = PipelineJob(name="replay", database=store) with job: a = plus_one.pod(make_source(REPLAY_ROWS), label="A") SelectDataColumns(["a"])(a, label="OUT") From d2784ed4b6f6b67f8bca7cfc96cae41ced933f18 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 19:09:30 -0800 Subject: [PATCH 8/9] fix(operator_node): a failed REPLAY waits for its input readers to end (ITL-1075) The REPLAY branch cancelled its drain tasks on a failure but returned without waiting for them, so the node raised with input readers still running. The finally block now awaits them (return_exceptions=True), so the replay error stays the error raised. Co-Authored-By: Claude Opus 5.5 --- src/orcapod/core/nodes/operator_node.py | 7 +++-- .../plans/2026-11-13-itl-1075-async-hang.md | 5 ++++ .../test_async_fan_out_does_not_hang.py | 26 +++++++++++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index 8effef7cb..074fdf285 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -1708,10 +1708,13 @@ async def async_execute( assert self._cached_output_stream is not None for tag, data in self._cached_output_stream.iter_data(): await output.send((tag, data)) - await asyncio.gather(*discards) + _ = await asyncio.gather(*discards) finally: for discard in discards: - discard.cancel() + _ = discard.cancel() + # Wait for the readers to end; a failure of the replay stays the + # error raised. + _ = await asyncio.gather(*discards, return_exceptions=True) ctx_obs.on_node_end(node_label, node_hash) return # finally block closes output diff --git a/superpowers/plans/2026-11-13-itl-1075-async-hang.md b/superpowers/plans/2026-11-13-itl-1075-async-hang.md index 7206bb2ff..915bba792 100644 --- a/superpowers/plans/2026-11-13-itl-1075-async-hang.md +++ b/superpowers/plans/2026-11-13-itl-1075-async-hang.md @@ -139,3 +139,8 @@ Findings: one inline Major (`semijoin.py` ~351), and two LanguageTool notes in t hand-over test never had a pending receive (right input closed before the start; instrumented: `pending=False`). Rewritten: the left reader signals its second receive, the right input closes only then (`pending=True`); dropping the hand-over row makes it fail (`[2, 4] != [2, 3, 4]`). +- 2026-11-14: merged origin/main a3a9757b (`b86ec44b`), `store=` -> `database=` in the fan-out tests + (`66c3e6c1`). CodeRabbit review 5901041898 of `66c3e6c1`: one body-only Minor (REPLAY drain + tasks cancelled but not awaited on failure). Reproduced; `test_failed_replay_leaves_no_input_reader_running` + failed, then the `finally` awaits the cancelled drains (`gather(..., return_exceptions=True)`). + Held locally: the coordinator paused pushes until main (broken at 5079f41a) is green again. diff --git a/tests/test_pipeline/test_async_fan_out_does_not_hang.py b/tests/test_pipeline/test_async_fan_out_does_not_hang.py index b00e2c5c3..caa958d51 100644 --- a/tests/test_pipeline/test_async_fan_out_does_not_hang.py +++ b/tests/test_pipeline/test_async_fan_out_does_not_hang.py @@ -24,6 +24,7 @@ import pytest from orcapod import PipelineJob, function_pod +from orcapod.channels import Channel from orcapod.core.operators import ( DropDataColumns, Join, @@ -34,6 +35,7 @@ ) from orcapod.databases import InMemoryArrowDatabase from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.protocols.core_protocols import DataProtocol, TagProtocol from orcapod.sources import DictSource from orcapod.types import CacheMode @@ -212,3 +214,27 @@ def test_replay_operator_node_reads_its_input(buffer_size: int) -> None: expected = expected_job.nodes[label].as_table().sort_by([("k", "ascending")]) assert got.num_rows == REPLAY_ROWS assert got.equals(expected) + + +class _FailingOutput: + """A writable channel whose sends fail: it stands for a failed downstream node.""" + + async def send(self, item: tuple[TagProtocol, DataProtocol]) -> None: + raise RuntimeError("downstream failed") + + async def close(self) -> None: + pass + + +def test_failed_replay_leaves_no_input_reader_running() -> None: + """When a REPLAY node fails, its input readers have ended by the time it raises.""" + + async def run() -> set[asyncio.Task[Any]]: + node = replay_job(InMemoryArrowDatabase()).nodes["OUT"] + upstream: Channel[tuple[TagProtocol, DataProtocol]] = Channel(buffer_size=4) + # The upstream stays open, so the node's reader of it is still waiting. + with pytest.raises(RuntimeError, match="downstream failed"): + await node.async_execute([upstream.reader], _FailingOutput()) + return asyncio.all_tasks() - {asyncio.current_task()} + + assert asyncio.run(asyncio.wait_for(run(), RUN_TIMEOUT)) == set() From e9dcf94eca5cbd2685fcea67742776e3e780a05f Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Fri, 13 Nov 2026 19:41:04 -0800 Subject: [PATCH 9/9] test(async): cleanup tests fail their input while a reader waits; 3.11-safe (ITL-1075) The REPLAY cleanup test failed the replay before its drain task had started, so it checked the cleanup of a reader that never read; the output now fails only once the reader is waiting on the open channel. The SemiJoin right-error test likewise fails the right input only once a left receive waits on the empty left channel. Both tests assert that no task is left; they now bound the run with asyncio.timeout, since on Python 3.11 asyncio.wait_for runs the coroutine in a task of its own (CI failed on 3.11). The plan's Task 2 example is marked superseded by the review-round fix. Co-Authored-By: Claude Opus 5.5 --- .../plans/2026-11-13-itl-1075-async-hang.md | 9 ++++ .../test_native_async_operators.py | 23 ++++++--- .../test_async_fan_out_does_not_hang.py | 47 ++++++++++++++++--- 3 files changed, 65 insertions(+), 14 deletions(-) diff --git a/superpowers/plans/2026-11-13-itl-1075-async-hang.md b/superpowers/plans/2026-11-13-itl-1075-async-hang.md index 915bba792..106ad43b5 100644 --- a/superpowers/plans/2026-11-13-itl-1075-async-hang.md +++ b/superpowers/plans/2026-11-13-itl-1075-async-hang.md @@ -60,6 +60,11 @@ intermediate channel at a fixed 64) before anything blocks. **Files:** Modify `src/orcapod/core/operators/semijoin.py` (`async_execute`) +> **Superseded** by "CodeRabbit review round" below: this loop stays blocked in +> `left_ch.receive()` when the right input ends meanwhile. The shipped code awaits each left +> receive as a task together with the right task, and hands a receive still pending to the +> left rows (`_LeftRows`). The example is kept as the original plan. + ```python async with asyncio.TaskGroup() as tg: right_task = tg.create_task(right_ch.collect()) @@ -144,3 +149,7 @@ Findings: one inline Major (`semijoin.py` ~351), and two LanguageTool notes in t tasks cancelled but not awaited on failure). Reproduced; `test_failed_replay_leaves_no_input_reader_running` failed, then the `finally` awaits the cancelled drains (`gather(..., return_exceptions=True)`). Held locally: the coordinator paused pushes until main (broken at 5079f41a) is green again. +- 2026-11-14: CodeRabbit review 5901263093 of `256b230d` and CI: Task 2 example marked superseded; + the REPLAY cleanup test now fails the replay only once the drain is reading (it was cancelled + before starting); `asyncio.timeout` instead of `asyncio.wait_for` in the two tests that assert + on `asyncio.all_tasks()` (on 3.11 `wait_for` runs the coroutine in an extra task: CI 3.11 failed). diff --git a/tests/test_channels/test_native_async_operators.py b/tests/test_channels/test_native_async_operators.py index 2ed3b3827..cd2e17c68 100644 --- a/tests/test_channels/test_native_async_operators.py +++ b/tests/test_channels/test_native_async_operators.py @@ -1061,9 +1061,16 @@ def _id_stream(ids: list[int], column: str) -> ArrowTableStream: class _FailingChannel: - """A readable channel whose reading fails: it stands for a failed right input.""" + """A readable channel whose reading fails: it stands for a failed right input. + + A read fails once ``after`` is set. + """ + + def __init__(self, after: asyncio.Event) -> None: + self._after = after async def receive(self) -> _Row: + _ = await self._after.wait() raise RuntimeError("right input failed") def __aiter__(self) -> "_FailingChannel": @@ -1147,17 +1154,19 @@ async def test_kept_matching_row_is_emitted_while_left_is_open(self): async def test_right_input_error_surfaces_while_left_is_open(self): left = _id_stream([2], "a") left_ch, output_ch = Channel[_Row](4), Channel[_Row](4) + left_reader = _SignallingReader(left_ch, signal_on_call=2) await left_ch.writer.send(next(iter(left.iter_data()))) + # The right input fails once the second left receive waits on the open, + # empty left channel. ``asyncio.timeout``, unlike ``asyncio.wait_for`` on + # 3.11, adds no task. with pytest.raises(RuntimeError, match="right input failed"): - await asyncio.wait_for( - SemiJoin().async_execute( - [left_ch.reader, _FailingChannel()], + async with asyncio.timeout(_HANG_GUARD_SECONDS): + await SemiJoin().async_execute( + [left_reader, _FailingChannel(after=left_reader.waiting)], output_ch.writer, input_pipeline_hashes=hashes(left, left), - ), - _HANG_GUARD_SECONDS, - ) + ) assert await output_ch.reader.collect() == [] # the output is closed # Nothing is left reading the left input. assert asyncio.all_tasks() == {asyncio.current_task()} diff --git a/tests/test_pipeline/test_async_fan_out_does_not_hang.py b/tests/test_pipeline/test_async_fan_out_does_not_hang.py index caa958d51..074ca2726 100644 --- a/tests/test_pipeline/test_async_fan_out_does_not_hang.py +++ b/tests/test_pipeline/test_async_fan_out_does_not_hang.py @@ -216,10 +216,39 @@ def test_replay_operator_node_reads_its_input(buffer_size: int) -> None: assert got.equals(expected) +class _ReaderThatSignals: + """Reads a channel and sets ``reading`` when a read starts.""" + + def __init__(self, channel: Channel[tuple[TagProtocol, DataProtocol]]) -> None: + self._reader = channel.reader + self.reading = asyncio.Event() + + async def receive(self) -> tuple[TagProtocol, DataProtocol]: + self.reading.set() + return await self._reader.receive() + + def __aiter__(self) -> "_ReaderThatSignals": + return self + + async def __anext__(self) -> tuple[TagProtocol, DataProtocol]: + self.reading.set() + return await self._reader.__anext__() + + async def collect(self) -> list[tuple[TagProtocol, DataProtocol]]: + return [row async for row in self] + + class _FailingOutput: - """A writable channel whose sends fail: it stands for a failed downstream node.""" + """A writable channel whose sends fail: it stands for a failed downstream node. + + A send fails only once ``after`` is set. + """ + + def __init__(self, after: asyncio.Event) -> None: + self._after = after async def send(self, item: tuple[TagProtocol, DataProtocol]) -> None: + _ = await self._after.wait() raise RuntimeError("downstream failed") async def close(self) -> None: @@ -230,11 +259,15 @@ def test_failed_replay_leaves_no_input_reader_running() -> None: """When a REPLAY node fails, its input readers have ended by the time it raises.""" async def run() -> set[asyncio.Task[Any]]: - node = replay_job(InMemoryArrowDatabase()).nodes["OUT"] - upstream: Channel[tuple[TagProtocol, DataProtocol]] = Channel(buffer_size=4) - # The upstream stays open, so the node's reader of it is still waiting. - with pytest.raises(RuntimeError, match="downstream failed"): - await node.async_execute([upstream.reader], _FailingOutput()) + # ``asyncio.timeout``, unlike ``asyncio.wait_for`` on 3.11, adds no task. + async with asyncio.timeout(RUN_TIMEOUT): + node = replay_job(InMemoryArrowDatabase()).nodes["OUT"] + upstream: Channel[tuple[TagProtocol, DataProtocol]] = Channel(4) + reader = _ReaderThatSignals(upstream) + # The replay fails once the node's reader of the upstream is waiting on + # it: the upstream stays open. + with pytest.raises(RuntimeError, match="downstream failed"): + await node.async_execute([reader], _FailingOutput(after=reader.reading)) return asyncio.all_tasks() - {asyncio.current_task()} - assert asyncio.run(asyncio.wait_for(run(), RUN_TIMEOUT)) == set() + assert asyncio.run(run()) == set()