diff --git a/changelog.d/ITL-1075.md b/changelog.d/ITL-1075.md new file mode 100644 index 000000000..58316a5f5 --- /dev/null +++ b/changelog.d/ITL-1075.md @@ -0,0 +1,4 @@ +### Fixed + +- An async job run no longer hangs when one node feeds both inputs of a `SemiJoin` (ITL-1075). `SemiJoin` read its left input only after its right input had ended. When a node fed both inputs (for example `SemiJoin(C, A)`, with `C` computed from `A`), the bounded channel into the unread left input filled, every node between `A` and that channel blocked in turn, `A` could not finish, and the run waited forever. A computing function node absorbs its whole input, which hid this on a first run: the hang showed on a run that found every row stored, or on any run of a graph of operators only. `SemiJoin` now reads both inputs at once. It keeps the left rows that arrive before the right input ends (at most the whole left input, which the sync path and `MergeJoin` already hold), and then streams the rest of the left input as before. Output rows, their order, system tags and hashes are unchanged. +- An async job run no longer hangs on an operator node in `CacheMode.REPLAY` with more upstream rows than a channel holds (ITL-1075). The node served its stored rows without reading its input, so its upstream blocked on a full channel. It now reads and discards its input while it replays, so the upstream runs over every row, as it does with the sync orchestrator. diff --git a/design_issues/ITL-1075-AH1.md b/design_issues/ITL-1075-AH1.md new file mode 100644 index 000000000..9d873ad61 --- /dev/null +++ b/design_issues/ITL-1075-AH1.md @@ -0,0 +1,33 @@ +## `src/orcapod/core/operators/` — Async execution + +### AH1 — An async run deadlocked when a consumer stopped reading one of its inputs +**Status:** resolved +**Severity:** high +**Issue:** ITL-1075 + +The async orchestrator joins nodes with bounded channels, so every node must keep reading +each input channel to its end: a consumer that waits on something before it reads an input +blocks that input's producer once the channel is full. Two consumers did: + +- `SemiJoin.async_execute` collected its right input before it read its left one. When one + node `A` fed both inputs (`SemiJoin(C, A)` with `C` computed from `A`, or `SemiJoin(A, C)`, + or `SemiJoin(A, B)` with `A` and `B` from one source), the channel into the unread left + input filled, every node between `A` and it blocked, `A` could not finish its right-side + rows, and the run waited forever. A computing function node spawns one task per input row + without a limit (no `max_concurrency`) and so absorbs its whole input, which hid the cycle + on a first run; a run that found every row stored, or a graph of operators only, hung once + the rows exceeded what the channels in the cycle hold (about 3 x `buffer_size` + 130). +- `OperatorNode.async_execute` in `CacheMode.REPLAY` served its stored rows without reading + its inputs, so any upstream with more rows than a channel holds blocked forever. + +Every other `async_execute` (the unary operators, `Join`, `MergeJoin` and the barrier +default, function, side-effect and source nodes) reads every input to its end. + +**Fix:** `SemiJoin` reads its left input while a task collects its right one +(`_collect_right_reading_left`), keeping the left rows that arrive before the right input +ends (at most the whole left input, as the sync path holds); a `REPLAY` operator node +reads and discards its inputs while it sends its stored rows. Each left receive is +awaited together with the right task, so the end or failure of the right input is acted on +at once; a receive still pending then is handed to the left rows, never cancelled while its +row may be read. Regression tests in `tests/test_pipeline/test_async_fan_out_does_not_hang.py` +and `tests/test_channels/test_native_async_operators.py`. diff --git a/src/orcapod/core/nodes/operator_node.py b/src/orcapod/core/nodes/operator_node.py index bf2517c92..ff4ca81b7 100644 --- a/src/orcapod/core/nodes/operator_node.py +++ b/src/orcapod/core/nodes/operator_node.py @@ -79,6 +79,16 @@ pc = LazyModule("pyarrow.compute") +async def _discard(channel: ReadableChannel[Any]) -> None: + """Read ``channel`` to its end, keeping nothing. + + Args: + channel: The channel to drain. + """ + async for _ in channel: + pass + + # --------------------------------------------------------------------------- # OperatorNodeBase — shared base (no DB) # --------------------------------------------------------------------------- @@ -1680,7 +1690,8 @@ async def async_execute( Without a database, or with one in OFF mode, nothing is stored. With a database: - - REPLAY: emit from DB, close output. + - REPLAY: emit from DB while reading and discarding every input, so + the upstream nodes run as with the sync orchestrator; close output. - OFF: delegate to operator, forward and keep results. - LOG: delegate to operator, forward and keep results, then store them in the DB. @@ -1708,10 +1719,23 @@ async def async_execute( self._pipeline_database is not None and self._cache_mode == CacheMode.REPLAY ): - self._replay_from_cache() - assert self._cached_output_stream is not None - for tag, data in self._cached_output_stream.iter_data(): - await output.send((tag, data)) + # Read and discard every input while the stored rows are sent: the + # upstream nodes run over every row, as with the sync orchestrator, + # and an input no one reads blocks its producer, and the run, once + # its channel is full (ITL-1075). + discards = [asyncio.create_task(_discard(ch)) for ch in inputs] + try: + self._replay_from_cache() + assert self._cached_output_stream is not None + for tag, data in self._cached_output_stream.iter_data(): + await output.send((tag, data)) + _ = await asyncio.gather(*discards) + finally: + for discard in discards: + _ = discard.cancel() + # Wait for the readers to end; a failure of the replay stays the + # error raised. + _ = await asyncio.gather(*discards, return_exceptions=True) ctx_obs.on_node_end(node_label, node_hash) return # finally block closes output diff --git a/src/orcapod/core/operators/semijoin.py b/src/orcapod/core/operators/semijoin.py index 573efd29a..13e043aff 100644 --- a/src/orcapod/core/operators/semijoin.py +++ b/src/orcapod/core/operators/semijoin.py @@ -1,8 +1,11 @@ +import asyncio +import contextlib import warnings -from collections.abc import Collection, Sequence -from typing import TYPE_CHECKING, Any +from collections import deque +from collections.abc import AsyncGenerator, AsyncIterator, Collection, Sequence +from typing import TYPE_CHECKING, Any, final -from orcapod.channels import ReadableChannel, WritableChannel +from orcapod.channels import ChannelClosed, ReadableChannel, WritableChannel from orcapod.core.operators.base import BinaryOperator, require_shared_arrow_type from orcapod.core.operators.key_nullability import reconcile_key_nullability from orcapod.core.operators.match_keys import frozen_scalar @@ -293,9 +296,16 @@ async def async_execute( """Build-probe: collect right input, then stream left through a hash lookup. Phase 1 — Build: collect all rows from the right (filter) channel and - index them by the common-key values. - Phase 2 — Probe: stream left rows one at a time; for each row whose - common-key values appear in the right-side index, emit immediately. + index them by the common-key values. The left channel is read at the + same time (ITL-1075): the left rows that arrive before the right input + ends are kept, at most the whole left input, as the sync path and + ``MergeJoin``'s barrier path hold their inputs. Reading the left input + only after the right one ended deadlocked a run in which one node fed + both inputs. + Phase 2 — Probe: the kept left rows, then the rest of the left input + one at a time with backpressure; for each row whose common-key values + appear in the right-side index, emit immediately. Left rows keep their + arrival order. Falls back to barrier mode when the right input is empty (schema cannot be inferred from data) or when there are no common keys. @@ -312,11 +322,12 @@ async def async_execute( their system-tag names, so no column is named after an input's pipeline hash. """ + left_rows: _LeftRows | None = None try: - left_ch, right_ch = inputs[0], inputs[1] - - # Phase 1: Build right-side lookup - right_rows = await right_ch.collect() + # Phase 1: Build right-side lookup, reading the left input meanwhile + right_rows, left_rows = await _collect_right_reading_left( + inputs[0], inputs[1] + ) if not right_rows: # Empty right: determine common keys from the validated @@ -325,10 +336,11 @@ async def async_execute( common = self._common_keys_from_schema() if common: # Drain left channel (discard) — result is empty - await left_ch.collect() + async for _ in left_rows: + pass return # No common keys — pass all left rows through unchanged - async for tag, data in left_ch: + async for tag, data in left_rows: await output.send((tag, data)) return @@ -339,7 +351,7 @@ async def async_execute( common_keys: tuple[str, ...] | None = None right_lookup: set[tuple[object, ...]] = set() - async for tag, data in left_ch: + async for tag, data in left_rows: if common_keys is None: # First left row — determine common keys (the shared tag # columns, ITL-1088) and build index @@ -359,7 +371,7 @@ async def async_execute( if not common_keys: # No common keys — pass all left rows through await output.send((tag, data)) - async for t, p in left_ch: + async for t, p in left_rows: await output.send((t, p)) return @@ -384,6 +396,8 @@ async def async_execute( await output.send((tag, data)) finally: await output.close() + if left_rows is not None: + await left_rows.aclose() @staticmethod def _key_table(tag: TagProtocol, data: DataProtocol, key: str) -> "pa.Table": @@ -431,3 +445,131 @@ def _match_key( def identity_structure(self) -> Any: return self.__class__.__name__ + + +_Row = tuple[TagProtocol, DataProtocol] + + +async def _collect_right_reading_left( + left: ReadableChannel[_Row], + right: ReadableChannel[_Row], +) -> tuple[list[_Row], "_LeftRows"]: + """Collect ``right`` while reading ``left``, so neither input stops being read. + + A task collects ``right``; meanwhile the rows of ``left`` are read and kept, until + ``right`` ends or ``left`` does. A semi-join that waited for its right input before + reading its left one blocked a node feeding both inputs: the channel into the left + input filled, that node could not finish its right-side rows, and the run waited + forever (ITL-1075). + + Each left receive runs as a task, awaited together with the right one, so the end + (or failure) of ``right`` is acted on at once, not when ``left`` next sends or + closes. A receive still pending then is handed to the returned rows, which await + it in its place: it is never cancelled while its row may still be read. + + Args: + left: The left input channel. + right: The right input channel. + + Returns: + The rows of ``right``, and every row of ``left`` in arrival order: the rows + kept while ``right`` was collected, the pending receive's row, then the rest + of ``left`` as it arrives. + + Raises: + Exception: Whatever collecting ``right`` raises. A pending left receive is + then cancelled, since the left rows are not going to be read. + """ + right_task = asyncio.create_task(right.collect()) + kept: deque[_Row] = deque() + receive: asyncio.Task[_Row] | None = None + left_ended = False + try: + while not right_task.done(): + if receive is None: + receive = asyncio.create_task(left.receive()) + _ = await asyncio.wait( + (receive, right_task), return_when=asyncio.FIRST_COMPLETED + ) + if receive.done(): + finished, receive = receive, None + try: + kept.append(finished.result()) + except ChannelClosed: + left_ended = True + break + right_rows = await right_task + except BaseException: + if receive is not None: + await _cancel_receive(receive) + raise + finally: + if not right_task.done(): + _ = right_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await right_task + return right_rows, _LeftRows(left, kept, receive, left_ended) + + +async def _cancel_receive(receive: "asyncio.Task[_Row]") -> None: + """Cancel a left receive whose row is not going to be read, and wait for it to end. + + Args: + receive: The receive task; it may already be done. + """ + _ = receive.cancel() + _ = await asyncio.wait((receive,)) + if not receive.cancelled(): + # Retrieve its outcome (a row, or ``ChannelClosed``) so it is not reported. + _ = receive.exception() + + +@final +class _LeftRows: + """Every row of the left input in arrival order, for one pass. + + The rows kept while the right input was collected come first, then the row of + the receive that was pending when it ended, then the rest of the left channel. + """ + + def __init__( + self, + left: ReadableChannel[_Row], + kept: deque[_Row], + pending: "asyncio.Task[_Row] | None", + left_ended: bool, + ) -> None: + self._left = left + self._kept = kept + self._pending = pending + self._left_ended = left_ended + self._rows = self._iterate() + + def __aiter__(self) -> AsyncIterator[_Row]: + return self._rows + + async def _iterate(self) -> AsyncGenerator[_Row, None]: + while self._kept: + yield self._kept.popleft() + if self._pending is not None: + try: + row = await self._pending + except ChannelClosed: + return + finally: + self._pending = None + yield row + if not self._left_ended: + async for row in self._left: + yield row + + async def aclose(self) -> None: + """Release the left input when its rows are not all read. + + A pending receive is cancelled. After a complete pass there is none, so this + does nothing. + """ + await self._rows.aclose() + if self._pending is not None: + pending, self._pending = self._pending, None + await _cancel_receive(pending) diff --git a/superpowers/plans/2026-11-13-itl-1075-async-hang.md b/superpowers/plans/2026-11-13-itl-1075-async-hang.md new file mode 100644 index 000000000..106ad43b5 --- /dev/null +++ b/superpowers/plans/2026-11-13-itl-1075-async-hang.md @@ -0,0 +1,155 @@ +# ITL-1075 Async job run hangs (bounded-channel deadlock) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use sensei:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** An async `PipelineJob.run()` never deadlocks when one node feeds both inputs of a later +`SemiJoin` (or an operator node in `CacheMode.REPLAY` has upstreams), on any run and any `buffer_size`. + +**Architecture:** The deadlock is a consumer that stops reading one input while it waits for +something only that input's producer can unblock. On `main` two consumers do that: +`SemiJoin.async_execute` (collects the right input before reading the left) and +`OperatorNode.async_execute` in `REPLAY` (never reads its inputs). Fix each in place: `SemiJoin` +reads the left input while a task collects the right; a `REPLAY` node drains (discards) its +inputs while it replays. No orchestrator, channel or `buffer_size` change. + +**Tech Stack:** asyncio, pytest (`asyncio.run(asyncio.wait_for(job.run_async(...)))` as the guard). + +Branch: `eywalker/itl-1075-async-job-run-hangs-on-a-re-run-when-one` from `origin/main` `163e5ddc`. + +--- + +## Cause (established, evidence in the coordinator message) + +Task dump of the hung run 2 (1000 rows, buffer 64), each task and the queue it waits on: + +- `SJ` (SemiJoin) `collect` of its right input, `get` on edge A -> SJ (0/64): waits for A to close. +- `A` `route_inputs`, `put` on edge A -> J (64/64): the broadcast writer delivers to J's queue first. +- `J` drain tasks `put` on Join's merged queue (64/64); Join loop `put` on J's intermediate (64/64); + J forward `put` on edge J -> C (64/64). +- `C` `route_inputs`, `put` on edge C -> SJ (64/64): SJ reads its left input only after the right closed. + +Cycle: SJ waits A, A waits J, J waits C, C waits SJ. + +Run 1 passes because a computing function node runs its pod's `async_execute`, which (no +`max_concurrency`) spawns one task per input row with no limit: C absorbs every row J emits, +so J never blocks. Proof: capping `max_concurrency` at 4 makes run 1 hang the same way. On a +cache-hit run `route_inputs` sends hits straight to the output with a blocking send. + +300 rows pass at buffer 64 because the cycle's queues hold about 3 x buffer + 130 rows +(A -> J, J -> C, C -> SJ at `buffer_size`, plus Join's merged queue and the operator node's +intermediate channel at a fixed 64) before anything blocks. + +## Tasks + +### Task 1: regression tests (fail on main by timeout) + +**Files:** Create `tests/test_pipeline/test_async_fan_out_does_not_hang.py` + +- Operators-only graph: `A = MapData(x->a)(src)`, `B = MapData(x->b)(src)`, + `C = MapData(b->c)(DropDataColumns(a)(Join(A, B)))`, `OUT = op(C, A)` for op in + Join / MergeJoin / SemiJoin, and `SemiJoin(A, C)` (fan-out node on the left); + 1000 rows; `buffer_size` 1, 16, 64; runs 1 and 2; each run under + `asyncio.wait_for(job.run_async(...), timeout)`; rows equal a sync job's rows. +- Issue graph with function pods (fa, fb, fc), 300 rows, buffer 1: run 1 and run 2 finish and equal sync. +- REPLAY operator node with 200 rows upstream, buffer 1: run finishes (only if the coordinator + keeps REPLAY in scope). +- [ ] Run, confirm SemiJoin cases (and REPLAY) fail with `TimeoutError` on main. +- [ ] WIP commit. + +### Task 2: SemiJoin reads both inputs at once + +**Files:** Modify `src/orcapod/core/operators/semijoin.py` (`async_execute`) + +> **Superseded** by "CodeRabbit review round" below: this loop stays blocked in +> `left_ch.receive()` when the right input ends meanwhile. The shipped code awaits each left +> receive as a task together with the right task, and hands a receive still pending to the +> left rows (`_LeftRows`). The example is kept as the original plan. + +```python +async with asyncio.TaskGroup() as tg: + right_task = tg.create_task(right_ch.collect()) + early_left = [] + left_closed = False + while not right_task.done(): + try: + early_left.append(await left_ch.receive()) + except ChannelClosed: + left_closed = True + break +right_rows = right_task.result() + +async def left_rows(): + for row in early_left: + yield row + if not left_closed: + async for row in left_ch: + yield row +``` + +The rest of the method iterates `left_rows()` where it iterated `left_ch`. Output order stays +the left arrival order. Memory: left rows that arrive before the right input closes are held +(at most the whole left input, as the barrier `MergeJoin` holds both inputs and the sync path +holds both tables); after that the left streams with backpressure. + +- [ ] Run Task 1 tests and `tests/test_core/operators/test_async_sync_equivalence.py`, SemiJoin tests. +- [ ] Commit `fix(semijoin): read both inputs at once so a shared upstream cannot deadlock (ITL-1075)`. + +### Task 3 (if approved): REPLAY operator node drains its inputs + +**Files:** Modify `src/orcapod/core/nodes/operator_node.py` (`async_execute`, REPLAY branch): +drain each input channel (`async for _ in ch: pass`) in a task group alongside the replay. + +### Task 4: docs, changelog, design issue + +- `changelog.d/ITL-1075.md` (`### Fixed`). +- `design_issues/ITL-1075-AH1.md` under `## src/orcapod/core/operators/ — Async execution`. +- `uv run python scripts/assemble_changelog.py --check`, `uv run python scripts/design_issues.py --check`. + +### Task 5: verify, PR + +- ruff check/format, basedpyright (exit code), targeted pytest (one process, py3.12), golden tests unchanged. +- Merge origin/main, push, PR `Fixes ITL-1075`, CodeRabbit rounds, CI green on 3.11/3.12/3.13. + +## Progress notes + +- 2026-11-13: cause established (task dump + concurrency cap experiment); shape matrix measured. + +## CodeRabbit review round (review 5900648222 of `f8dd3163`, 2026-11-14) + +Findings: one inline Major (`semijoin.py` ~351), and two LanguageTool notes in the review body. + +- Major: `_collect_right_reading_left` checks `right_task.done()` only before awaiting + `left.receive()`. Reproduced (scratch script): with one matching left row sent and the left + channel open, right closed, SemiJoin emits nothing until the left sends again or closes; a + `right.collect()` that raises surfaces only after the left closes. No pipeline deadlock is + reachable with today's consumers (every consumer reads all of its inputs at once), so the effect + is delayed probing and a delayed right error. Fix: + - wait on the pending left receive (a task) and `right_task` together + (`asyncio.wait(..., FIRST_COMPLETED)`); a finished receive's row is kept (or `ChannelClosed` + ends the left); when `right_task` is done, stop; + - never cancel the pending receive on the success path: the left iterator yields the kept rows, + then awaits the pending receive, then reads the rest, so the left arrival order is kept; + - on a failure (right raises, cancellation, a probe error) the pending receive is cancelled: + the run is failing and the left rows are discarded anyway; + - tests (event-controlled, `asyncio.wait_for` only as a hang guard, no sleeps) in + `tests/test_channels/test_native_async_operators.py`: matching row emitted while left open; + right-side error raised while left open. +- LanguageTool "that" -> "who" (design issue line 9, plan line 8): "consumer" is a coroutine, + not a person; decline. +- 2026-11-14: coordinator approved the round plan, plus a test that a left row received by the + pending receive when the right ends is neither lost nor duplicated. Three tests in + `TestSemiJoinRightEndsWhileLeftOpen` failed with `TimeoutError` on `f8dd3163`; pass after the + fix (`_LeftRows` holds the pending receive); equivalence and the 15 fan-out cases rerun green. +- 2026-11-14: CodeRabbit review 5900799249 of `a63d84ea` (one body-only Trivial nitpick): the + hand-over test never had a pending receive (right input closed before the start; instrumented: + `pending=False`). Rewritten: the left reader signals its second receive, the right input closes + only then (`pending=True`); dropping the hand-over row makes it fail (`[2, 4] != [2, 3, 4]`). +- 2026-11-14: merged origin/main a3a9757b (`b86ec44b`), `store=` -> `database=` in the fan-out tests + (`66c3e6c1`). CodeRabbit review 5901041898 of `66c3e6c1`: one body-only Minor (REPLAY drain + tasks cancelled but not awaited on failure). Reproduced; `test_failed_replay_leaves_no_input_reader_running` + failed, then the `finally` awaits the cancelled drains (`gather(..., return_exceptions=True)`). + Held locally: the coordinator paused pushes until main (broken at 5079f41a) is green again. +- 2026-11-14: CodeRabbit review 5901263093 of `256b230d` and CI: Task 2 example marked superseded; + the REPLAY cleanup test now fails the replay only once the drain is reading (it was cancelled + before starting); `asyncio.timeout` instead of `asyncio.wait_for` in the two tests that assert + on `asyncio.all_tasks()` (on 3.11 `wait_for` runs the coroutine in an extra task: CI 3.11 failed). diff --git a/tests/test_channels/test_native_async_operators.py b/tests/test_channels/test_native_async_operators.py index 68d681681..cd2e17c68 100644 --- a/tests/test_channels/test_native_async_operators.py +++ b/tests/test_channels/test_native_async_operators.py @@ -27,7 +27,7 @@ import pyarrow as pa import pytest -from orcapod.channels import Channel +from orcapod.channels import Channel, ChannelClosed from orcapod.core.operators import ( Batch, DropDataColumns, @@ -40,7 +40,7 @@ SemiJoin, ) from orcapod.core.streams.arrow_table_stream import ArrowTableStream -from orcapod.protocols.core_protocols import StreamProtocol +from orcapod.protocols.core_protocols import DataProtocol, StreamProtocol, TagProtocol from orcapod.system_constants import constants from orcapod.types import ContentHash @@ -1041,6 +1041,177 @@ async def test_large_input_streaming(self): assert result_ids == expected_ids +# =================================================================== +# SemiJoin — the right input ends while the left one is still open (ITL-1075) +# =================================================================== + + +def _id_stream(ids: list[int], column: str) -> ArrowTableStream: + """Stream with an ``id`` tag and one int64 data column named ``column``.""" + table = pa.table( + { + "id": pa.array(ids, type=pa.int64()), + column: pa.array([i * 10 for i in ids], type=pa.int64()), + } + ) + return ArrowTableStream(table, tag_columns=["id"]) + + +_Row = tuple[TagProtocol, DataProtocol] + + +class _FailingChannel: + """A readable channel whose reading fails: it stands for a failed right input. + + A read fails once ``after`` is set. + """ + + def __init__(self, after: asyncio.Event) -> None: + self._after = after + + async def receive(self) -> _Row: + _ = await self._after.wait() + raise RuntimeError("right input failed") + + def __aiter__(self) -> "_FailingChannel": + return self + + async def __anext__(self) -> _Row: + return await self.receive() + + async def collect(self) -> list[_Row]: + return [await self.receive()] + + +class _SignallingReader: + """Reads a channel and sets ``waiting`` when the given receive call starts.""" + + def __init__(self, channel: Channel[_Row], signal_on_call: int) -> None: + self._reader = channel.reader + self._signal_on_call = signal_on_call + self._calls = 0 + self.waiting = asyncio.Event() + + async def receive(self) -> _Row: + self._calls += 1 + if self._calls == self._signal_on_call: + self.waiting.set() + return await self._reader.receive() + + def __aiter__(self) -> "_SignallingReader": + return self + + async def __anext__(self) -> _Row: + try: + return await self.receive() + except ChannelClosed: + raise StopAsyncIteration + + async def collect(self) -> list[_Row]: + return [row async for row in self] + + +# Guards a wait that must end at once; a timeout means the operator hung. +_HANG_GUARD_SECONDS = 10 + + +class TestSemiJoinRightEndsWhileLeftOpen: + """The end of the right input is acted on at once, not at the next left row. + + Every step waits on a channel event; ``asyncio.wait_for`` only guards against a hang. + """ + + @pytest.mark.asyncio + async def test_kept_matching_row_is_emitted_while_left_is_open(self): + left = _id_stream([2, 7], "a") + right = _id_stream([2, 3], "b") + left_rows = list(left.iter_data()) + left_ch, right_ch, output_ch = ( + Channel[_Row](4), + Channel[_Row](4), + Channel[_Row](4), + ) + await left_ch.writer.send(left_rows[0]) + await feed(right, right_ch) + + task = asyncio.create_task( + SemiJoin().async_execute( + [left_ch.reader, right_ch.reader], + output_ch.writer, + input_pipeline_hashes=hashes(left, right), + ) + ) + tag, _ = await asyncio.wait_for(output_ch.reader.receive(), _HANG_GUARD_SECONDS) + assert tag.as_dict()["id"] == 2 + assert not task.done() # the left input is still open + + await left_ch.writer.send(left_rows[1]) + await left_ch.writer.close() + await asyncio.wait_for(task, _HANG_GUARD_SECONDS) + assert await output_ch.reader.collect() == [] + + @pytest.mark.asyncio + async def test_right_input_error_surfaces_while_left_is_open(self): + left = _id_stream([2], "a") + left_ch, output_ch = Channel[_Row](4), Channel[_Row](4) + left_reader = _SignallingReader(left_ch, signal_on_call=2) + await left_ch.writer.send(next(iter(left.iter_data()))) + + # The right input fails once the second left receive waits on the open, + # empty left channel. ``asyncio.timeout``, unlike ``asyncio.wait_for`` on + # 3.11, adds no task. + with pytest.raises(RuntimeError, match="right input failed"): + async with asyncio.timeout(_HANG_GUARD_SECONDS): + await SemiJoin().async_execute( + [left_reader, _FailingChannel(after=left_reader.waiting)], + output_ch.writer, + input_pipeline_hashes=hashes(left, left), + ) + assert await output_ch.reader.collect() == [] # the output is closed + # Nothing is left reading the left input. + assert asyncio.all_tasks() == {asyncio.current_task()} + + @pytest.mark.asyncio + async def test_left_row_in_flight_when_right_ends_is_kept_once_in_order(self): + # The right input ends while a receive on the empty left channel is pending. + # The row that receive then gets is emitted once, after the kept row and + # before the rows that follow it. + left = _id_stream([2, 3, 9, 4], "a") + right = _id_stream([2, 3, 4], "b") + left_rows = list(left.iter_data()) + left_ch, right_ch, output_ch = ( + Channel[_Row](4), + Channel[_Row](4), + Channel[_Row](4), + ) + left_reader = _SignallingReader(left_ch, signal_on_call=2) + await left_ch.writer.send(left_rows[0]) + for row in right.iter_data(): + await right_ch.writer.send(row) # the right input stays open for now + + task = asyncio.create_task( + SemiJoin().async_execute( + [left_reader, right_ch.reader], + output_ch.writer, + input_pipeline_hashes=hashes(left, right), + ) + ) + # The first left row is kept; the second receive waits on an empty channel. + _ = await asyncio.wait_for(left_reader.waiting.wait(), _HANG_GUARD_SECONDS) + await right_ch.writer.close() + first, _ = await asyncio.wait_for( + output_ch.reader.receive(), _HANG_GUARD_SECONDS + ) + for row in left_rows[1:]: + await left_ch.writer.send(row) + await left_ch.writer.close() + await asyncio.wait_for(task, _HANG_GUARD_SECONDS) + + rest = await output_ch.reader.collect() + ids = [first.as_dict()["id"]] + [tag.as_dict()["id"] for tag, _ in rest] + assert ids == [2, 3, 4] + + # =================================================================== # Join — native async # =================================================================== diff --git a/tests/test_pipeline/test_async_fan_out_does_not_hang.py b/tests/test_pipeline/test_async_fan_out_does_not_hang.py new file mode 100644 index 000000000..074ca2726 --- /dev/null +++ b/tests/test_pipeline/test_async_fan_out_does_not_hang.py @@ -0,0 +1,273 @@ +"""An async run never deadlocks when one node feeds both inputs of a later operator (ITL-1075). + +The async orchestrator joins nodes with bounded channels. ``SemiJoin`` collected its right +input before it read its left one. When one node fed both inputs, the channel into the +unread left input filled, every node between the shared node and that channel blocked in +turn, the shared node could not finish its right-side rows, and the run waited forever. +A function node that computes absorbs its whole input (one task per row), which hid the +deadlock on a first run; a run that finds every row stored streams with backpressure and +hung. A graph of operators only hangs on its first run. + +An operator node in ``CacheMode.REPLAY`` never read its inputs at all: with more upstream +rows than a channel holds, the upstream blocked and the run hung, with no fan-out. + +Each run here is awaited under ``asyncio.wait_for``: a deadlock fails the test with +``TimeoutError`` instead of hanging the suite. No test sleeps. +""" + +from __future__ import annotations + +import asyncio +import functools +from typing import Any + +import pytest + +from orcapod import PipelineJob, function_pod +from orcapod.channels import Channel +from orcapod.core.operators import ( + DropDataColumns, + Join, + MapData, + MergeJoin, + SelectDataColumns, + SemiJoin, +) +from orcapod.databases import InMemoryArrowDatabase +from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.protocols.core_protocols import DataProtocol, TagProtocol +from orcapod.sources import DictSource +from orcapod.types import CacheMode + +# Seconds one run may take. A run here takes well under two seconds; a deadlocked run +# never ends. +RUN_TIMEOUT = 30.0 + +N_ROWS = 1000 # far above every channel buffer in the cycle + +BUFFER_SIZES = [1, 16, 64] + + +def make_source(n_rows: int) -> DictSource: + """``n_rows`` rows tagged ``(g, k)`` with one data column ``x``.""" + rows = [{"g": i % 4, "k": i, "x": i} for i in range(n_rows)] + return DictSource(rows, tag_columns=["g", "k"], source_id="src") + + +def run_async_job(job: PipelineJob, buffer_size: int) -> None: + """Run ``job`` with the async orchestrator; fail with ``TimeoutError`` on a deadlock.""" + orchestrator = AsyncPipelineOrchestrator(buffer_size=buffer_size) + asyncio.run(asyncio.wait_for(job.run_async(orchestrator=orchestrator), RUN_TIMEOUT)) + + +def output_rows(job: PipelineJob) -> list[dict[str, Any]]: + """The rows of node ``OUT``, with system tags and source info, in a fixed order.""" + table = job.nodes["OUT"].as_table(columns={"system_tags": True, "source": True}) + names = sorted(table.column_names) + return ( + table.select(names).sort_by([(name, "ascending") for name in names]).to_pylist() + ) + + +# --------------------------------------------------------------------------- +# A graph of operators only: A feeds the join that makes C, and the last operator. +# --------------------------------------------------------------------------- + +LAST_OPERATORS = { + "Join(C, A)": lambda c, a: Join()(c, a, label="OUT"), + "MergeJoin(C, A)": lambda c, a: MergeJoin()(c, a, label="OUT"), + "SemiJoin(C, A)": lambda c, a: SemiJoin()(c, a, label="OUT"), + "SemiJoin(A, C)": lambda c, a: SemiJoin()(a, c, label="OUT"), +} + + +def operator_job(last: str) -> PipelineJob: + """``A`` and ``B`` from one source, ``C`` from ``Join(A, B)``, then ``last`` of C and A.""" + job = PipelineJob(name="fan_out", database=InMemoryArrowDatabase()) + with job: + src = make_source(N_ROWS) + a = MapData({"x": "a"})(src, label="A") + b = MapData({"x": "b"})(src, label="B") + joined = Join()(a, b, label="J") + c = MapData({"b": "c"})(DropDataColumns(["a"])(joined, label="D"), label="C") + LAST_OPERATORS[last](c, a) + return job + + +@functools.cache +def sync_operator_rows(last: str) -> list[dict[str, Any]]: + """The rows of ``operator_job(last)`` run once with the sync orchestrator.""" + job = operator_job(last) + job.run(orchestrator=SyncPipelineOrchestrator()) + return output_rows(job) + + +@pytest.mark.parametrize("buffer_size", BUFFER_SIZES) +@pytest.mark.parametrize("last", list(LAST_OPERATORS)) +def test_operator_fan_out_completes_on_both_runs(last: str, buffer_size: int) -> None: + """Runs 1 and 2 finish and give the rows of a sync run.""" + expected = sync_operator_rows(last) + assert expected # the comparison is not vacuous + + job = operator_job(last) + for _ in range(2): + run_async_job(job, buffer_size) + assert output_rows(job) == expected + + +# --------------------------------------------------------------------------- +# The graph of the issue: function pods A, B, C; SemiJoin(C, A). +# --------------------------------------------------------------------------- + + +@function_pod(output_keys="a") +def plus_one(x: int) -> int: + """Return ``x + 1``.""" + return x + 1 + + +@function_pod(output_keys="b") +def plus_two(x: int) -> int: + """Return ``x + 2``.""" + return x + 2 + + +@function_pod(output_keys="c") +def add(a: int, b: int) -> int: + """Return ``a + b``.""" + return a + b + + +def function_job(store: InMemoryArrowDatabase, n_rows: int) -> PipelineJob: + """The issue's graph: ``SemiJoin(C, A)`` where ``A`` also feeds ``C``.""" + job = PipelineJob(name="fan_out_pods", database=store) + with job: + src = make_source(n_rows) + a = plus_one.pod(src, label="A") + b = plus_two.pod(src, label="B") + c = add.pod(Join()(a, b, label="J"), label="C") + SemiJoin()(c, a, label="OUT") + return job + + +def test_function_fan_out_completes_when_every_row_is_stored() -> None: + """The second run, which finds every row stored, finishes with the rows of the first. + + 300 rows at buffer size 1 deadlocked on ``main``; 1,000 rows at 1, 16 and 64 are in + the measured table of the pull request (too slow for the default suite). + """ + expected_job = function_job(InMemoryArrowDatabase(), 300) + expected_job.run(orchestrator=SyncPipelineOrchestrator()) + expected = expected_job.nodes["OUT"].as_table() + + job = function_job(InMemoryArrowDatabase(), 300) + for _ in range(2): + run_async_job(job, buffer_size=1) + got = job.nodes["OUT"].as_table() + assert got.sort_by([("k", "ascending")]).equals( + expected.sort_by([("k", "ascending")]) + ) + + +# --------------------------------------------------------------------------- +# An operator node in REPLAY mode: it serves its stored rows and must still read its input. +# --------------------------------------------------------------------------- + +REPLAY_ROWS = 200 # above both buffer sizes tested + + +def replay_job(store: InMemoryArrowDatabase) -> PipelineJob: + """``A`` (a function pod) feeding operator node ``OUT``, which replays stored rows. + + ``OUT`` first runs once in ``CacheMode.LOG`` with the sync orchestrator, so it has + stored rows; then it is switched to ``CacheMode.REPLAY``. + """ + job = PipelineJob(name="replay", database=store) + with job: + a = plus_one.pod(make_source(REPLAY_ROWS), label="A") + SelectDataColumns(["a"])(a, label="OUT") + out = job.nodes["OUT"] + # attach_databases is the public way to turn on LOG / REPLAY mode. + out.attach_databases(pipeline_database=store.at("replay"), cache_mode=CacheMode.LOG) + job.run(orchestrator=SyncPipelineOrchestrator()) + out.attach_databases( + pipeline_database=store.at("replay"), cache_mode=CacheMode.REPLAY + ) + return job + + +@pytest.mark.parametrize("buffer_size", [1, 64]) +def test_replay_operator_node_reads_its_input(buffer_size: int) -> None: + """A REPLAY run finishes and does what a sync REPLAY run does: no more, no less. + + In both, the upstream ``A`` still runs over every row (the sync orchestrator executes + every node), and ``OUT`` gives its stored rows. + """ + expected_job = replay_job(InMemoryArrowDatabase()) + expected_job.run(orchestrator=SyncPipelineOrchestrator()) + + job = replay_job(InMemoryArrowDatabase()) + run_async_job(job, buffer_size) + + for label in ("A", "OUT"): + got = job.nodes[label].as_table().sort_by([("k", "ascending")]) + expected = expected_job.nodes[label].as_table().sort_by([("k", "ascending")]) + assert got.num_rows == REPLAY_ROWS + assert got.equals(expected) + + +class _ReaderThatSignals: + """Reads a channel and sets ``reading`` when a read starts.""" + + def __init__(self, channel: Channel[tuple[TagProtocol, DataProtocol]]) -> None: + self._reader = channel.reader + self.reading = asyncio.Event() + + async def receive(self) -> tuple[TagProtocol, DataProtocol]: + self.reading.set() + return await self._reader.receive() + + def __aiter__(self) -> "_ReaderThatSignals": + return self + + async def __anext__(self) -> tuple[TagProtocol, DataProtocol]: + self.reading.set() + return await self._reader.__anext__() + + async def collect(self) -> list[tuple[TagProtocol, DataProtocol]]: + return [row async for row in self] + + +class _FailingOutput: + """A writable channel whose sends fail: it stands for a failed downstream node. + + A send fails only once ``after`` is set. + """ + + def __init__(self, after: asyncio.Event) -> None: + self._after = after + + async def send(self, item: tuple[TagProtocol, DataProtocol]) -> None: + _ = await self._after.wait() + raise RuntimeError("downstream failed") + + async def close(self) -> None: + pass + + +def test_failed_replay_leaves_no_input_reader_running() -> None: + """When a REPLAY node fails, its input readers have ended by the time it raises.""" + + async def run() -> set[asyncio.Task[Any]]: + # ``asyncio.timeout``, unlike ``asyncio.wait_for`` on 3.11, adds no task. + async with asyncio.timeout(RUN_TIMEOUT): + node = replay_job(InMemoryArrowDatabase()).nodes["OUT"] + upstream: Channel[tuple[TagProtocol, DataProtocol]] = Channel(4) + reader = _ReaderThatSignals(upstream) + # The replay fails once the node's reader of the upstream is waiting on + # it: the upstream stays open. + with pytest.raises(RuntimeError, match="downstream failed"): + await node.async_execute([reader], _FailingOutput(after=reader.reading)) + return asyncio.all_tasks() - {asyncio.current_task()} + + assert asyncio.run(run()) == set()