From f9fa170f0c58501fc1503be2868defb0c5aee90e Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 16:40:48 -0800 Subject: [PATCH 1/6] fix(tests): make the second-interrupt test deterministic; list failures in the CI summary (ITL-1229) The Python 3.11 job of Run Tests on main failed on 3 of 6 pushes. Each failure is one failed test, test_second_interrupt_returns_and_a_late_function_writes_nothing[delta] (ITL-1192): the warning named nodes 'stuck', 'after' instead of 'stuck'. A row task of 'after' cancelled while in progress gets no end event, so its node stays counted as having a function that has not returned. The jobs looked silent because gh run view --log truncates long logs and -rsxX replaced pytest's default -rfE, so the closing summary never named the failed test. - The test holds every row of 'stuck' until the release, so no row of 'after' can be in progress when the cancellation lands; it still requires the warning to name only 'stuck'. - A new event-driven test pins the library imprecision as a strict xfail naming ITL-1192. - run-tests.yml passes -rfEsxX; run-tests.yml and release.yml set PYTHONFAULTHANDLER=1. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/release.yml | 5 + .github/workflows/run-tests.yml | 10 +- .../test_job_run_in_event_loop.py | 127 ++++++++++++++++-- 3 files changed, 130 insertions(+), 12 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a59649e1a..64c3f7584 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -68,6 +68,11 @@ jobs: run: uv sync --locked --all-extras --dev --python ${{ matrix.python-version }} - name: Run tests + # PYTHONFAULTHANDLER prints every thread's traceback if a process dies on a fatal + # signal, so a crash never ends the log silently (ITL-1229). Failures are listed in + # the closing summary by pytest's default -rfE. + env: + PYTHONFAULTHANDLER: "1" run: uv run --python ${{ matrix.python-version }} pytest -n logical -m "not postgres" --tb=short -q license-check: diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index dea3b3278..2c41a1971 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -54,8 +54,14 @@ jobs: - name: Run tests # In parallel on every core of the runner (pytest-xdist; ITL-1181); pytest-cov - # combines the workers' coverage into one report. - run: uv run --python ${{ matrix.python-version }} pytest -n logical -v -rsxX --durations=20 -m "not postgres" --cov=src --cov-report=term-missing --cov-report=xml + # combines the workers' coverage into one report. -rfEsxX lists failures and errors + # in the closing summary as well as skips and xfails: -r alone replaces pytest's + # default (fE), and a failure 25,000 lines up a log is easy to miss (ITL-1229). + # PYTHONFAULTHANDLER prints every thread's traceback if a process dies on a fatal + # signal, so a crash never ends the log silently. + env: + PYTHONFAULTHANDLER: "1" + run: uv run --python ${{ matrix.python-version }} pytest -n logical -v -rfEsxX --durations=20 -m "not postgres" --cov=src --cov-report=term-missing --cov-report=xml - name: Upload coverage reports to Codecov if: github.repository == 'metamorphic-labs/orcapod-python' diff --git a/tests/test_pipeline/test_job_run_in_event_loop.py b/tests/test_pipeline/test_job_run_in_event_loop.py index d989eca52..f870506de 100644 --- a/tests/test_pipeline/test_job_run_in_event_loop.py +++ b/tests/test_pipeline/test_job_run_in_event_loop.py @@ -17,12 +17,15 @@ from pathlib import Path from typing import Any +from typing_extensions import override + import pyarrow as pa import pytest from orcapod import PipelineJob, function_pod from orcapod.databases import DeltaTableDatabase, InMemoryArrowDatabase from orcapod.pipeline import AsyncPipelineOrchestrator, SyncPipelineOrchestrator +from orcapod.pipeline.observer import NoOpObserver from orcapod.sources import DictSource # Columns that differ between any two runs by design, also between two script runs: the @@ -285,12 +288,19 @@ def recorded(*args: Any, **kwargs: Any) -> Any: @function_pod(output_keys="y") def stuck_on_first_row(x: int) -> int: - """Row 0 presses the interrupt button twice and does not return until released.""" + """Row 0 presses the interrupt button twice; no row returns until released. + + Every row waits for the release, not only row 0: a row that returned would reach + ``after``, and a row of ``after`` still in progress when the cancellation lands is + named in the warning too (ITL-1192), so the nodes named would depend on timing + (ITL-1229). + """ stuck_calls.append(x) - if x == 0 and press_interrupt.is_set(): - _thread.interrupt_main() # first press: cancels the run, which then waits - # Second press, while the run waits for this function to return. - threading.Timer(1.0, _thread.interrupt_main).start() + if press_interrupt.is_set(): + if x == 0: + _thread.interrupt_main() # first press: cancels the run, which then waits + # Second press, while the run waits for the functions to return. + threading.Timer(1.0, _thread.interrupt_main).start() _ = stuck_release.wait(timeout=30) return x @@ -302,9 +312,14 @@ def after_stuck(y: int) -> int: return y + 1 +# Fewer rows than the smallest default thread pool (one CPU plus four threads), so every +# row of ``stuck_on_first_row`` has a thread while all of them wait for the release. +N_STUCK_ROWS = 4 + + def make_stuck_job(store: Any) -> PipelineJob: - """Six rows through ``stuck_on_first_row`` and then ``after_stuck``.""" - rows = [{"k": f"r{i}", "x": i} for i in range(6)] + """Four rows through ``stuck_on_first_row`` and then ``after_stuck``.""" + rows = [{"k": f"r{i}", "x": i} for i in range(N_STUCK_ROWS)] job = PipelineJob(name="stuck_demo", database=store) with job: source = DictSource( @@ -331,9 +346,9 @@ def test_second_interrupt_returns_and_a_late_function_writes_nothing( ) -> None: """A second interrupt returns at once; the function returning later stores nothing. - The first interrupt cancels the run, which then waits for row 0's function, still - running on a thread where it cannot be interrupted. The second interrupt returns - control with a warning naming the node. When the function returns afterwards, no + The first interrupt cancels the run, which then waits for the node's functions, still + running on threads where they cannot be interrupted. The second interrupt returns + control with a warning naming the node. When the functions return afterwards, no database write happens and no row reaches the next node, and row 0's result is never stored: a later run computes it. """ @@ -386,6 +401,98 @@ async def notebook_cell() -> None: assert 0 in stuck_calls +run_ended = threading.Event() +next_row_entered = threading.Event() +next_row_returned = threading.Event() + + +class RunEndSignal(NoOpObserver): + """Sets ``run_ended`` when the run ends: after a cancellation, once every row task is gone.""" + + @override + def on_run_end(self, run_id: str) -> None: + """Signal the end of the run.""" + run_ended.set() + + +def press_interrupt_once_next_row_returned() -> None: + """The second press, once the next node's function has returned.""" + if next_row_returned.wait(timeout=30): + _thread.interrupt_main() + + +@function_pod(output_keys="y") +def stuck_until_next_row_runs(x: int) -> int: + """Row 0 presses the interrupt button once a row of the next node is in its function.""" + if x == 0: + _ = next_row_entered.wait(timeout=30) + _thread.interrupt_main() # first press: cancels the run, which then waits + threading.Thread(target=press_interrupt_once_next_row_returned).start() + _ = stuck_release.wait(timeout=30) + return x + + +@function_pod(output_keys="z") +def returns_after_the_run_ended(y: int) -> int: + """In progress when the cancellation lands; returns once the cancelled run has ended.""" + next_row_entered.set() + _ = run_ended.wait(timeout=30) + next_row_returned.set() + return y + 1 + + +@pytest.mark.xfail( + strict=True, + raises=AssertionError, + reason=( + "ITL-1192: a row task cancelled while its function runs gets no end event, so " + "the warning names its node although the function has returned" + ), +) +@pytest.mark.usefixtures("interrupt_raises_keyboard_interrupt") +def test_second_interrupt_does_not_name_a_node_whose_function_returned() -> None: + """The warning names only the nodes whose functions have not returned. + + A row of ``after`` is in its function when the first interrupt cancels the run; the + function returns once the cancelled run has ended, and only then is the interrupt + pressed again, while row 0 of ``stuck`` still runs. Every step waits for the one + before, so the outcome does not depend on timing. Today the warning names ``after`` + too: a known defect (ITL-1192), pinned here so the fix flips this test. + """ + for event in (stuck_release, run_ended, next_row_entered, next_row_returned): + event.clear() + rows = [{"k": f"r{i}", "x": i} for i in range(N_STUCK_ROWS)] + job = PipelineJob(name="returned_demo", database=InMemoryArrowDatabase()) + with job: + source = DictSource( + data=rows, tag_columns=["k"], tag_only_columns=True, source_id="rows" + ) + stuck = stuck_until_next_row_runs.pod(source, label="stuck") + _ = returns_after_the_run_ended.pod(stuck, label="after") + + async def notebook_cell() -> None: + """A notebook cell: the job run inside the kernel's running loop.""" + _ = job.run(orchestrator=AsyncPipelineOrchestrator(), observer=RunEndSignal()) + + loop = asyncio.new_event_loop() + try: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + with pytest.raises(KeyboardInterrupt): + loop.run_until_complete(notebook_cell()) + finally: + loop.close() + stuck_release.set() + deadline = time.monotonic() + 30 + while bridge_threads() and time.monotonic() < deadline: + time.sleep(0.05) + + assert next_row_returned.is_set() + messages = [str(w.message) for w in caught if w.category is RuntimeWarning] + assert len(messages) == 1, messages + assert "but a function in node 'stuck' has not returned yet" in messages[0] + + async def async_stuck_on_first_row(x: int) -> int: """Row 0 presses the interrupt button twice and blocks the event loop until released.""" if x == 0: From d2bc556af0407fc10653c93ee7dd56e8cfb4b193 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 16:48:30 -0800 Subject: [PATCH 2/6] docs(design-issues): log the over-inclusive second-interrupt warning as INT1 (ITL-1192) Co-Authored-By: Claude Opus 5.5 --- docs/dev/design_issues/ITL-1192-INT1.md | 29 +++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 docs/dev/design_issues/ITL-1192-INT1.md diff --git a/docs/dev/design_issues/ITL-1192-INT1.md b/docs/dev/design_issues/ITL-1192-INT1.md new file mode 100644 index 000000000..173ee94c2 --- /dev/null +++ b/docs/dev/design_issues/ITL-1192-INT1.md @@ -0,0 +1,29 @@ +## `src/orcapod/pipeline/async_orchestrator.py` — interrupting a run inside a running loop + +### INT1 — the second-interrupt warning names a node whose cancelled row's function has returned +**Status:** open +**Severity:** low +**Issue:** ITL-1192 + +A job run from inside a running event loop (a notebook cell) runs on its own thread. The first +interrupt cancels it, and the second returns control with a `RuntimeWarning` naming the nodes +whose functions have not returned (`_warn_function_still_running`). The nodes come from +`_RowsInFlight`, which counts a row from `on_data_start` until `on_data_end` or +`on_data_crash`. A row task cancelled while in progress (`process_one` in +`FunctionPod.async_execute`, awaiting `run_in_executor`) gets `CancelledError` and fires +neither. So its node stays counted whether its function was queued, still running or had +already returned on its thread, and the warning can name too many nodes ("a function in +nodes 'stuck', 'after' has not returned yet", although `after`'s function returned long +before). The class docstring admits the queued case only. + +Effect: only the text of that warning, after a second interrupt. No crash and no wrong data: +the cancelled run writes nothing either way. The race made +`test_second_interrupt_returns_and_a_late_function_writes_nothing[delta]` flaky; it failed +half of the Python 3.11 runs on `main` (ITL-1229, which made that test deterministic). +`test_second_interrupt_does_not_name_a_node_whose_function_returned` +(`tests/test_pipeline/test_job_run_in_event_loop.py`) reproduces the defect without relying on +timing and is a strict xfail until it is fixed. + +Proposed fix: count a row as in flight from the worker thread. Wrap the callable given to +`run_in_executor` so the count rises when the function starts on its thread and falls when it +returns or raises, whatever happens to the asyncio task. Then remove the xfail marker. From 2ba04d8e2d19bec860b84c144b425a1d1eb3ecda Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 17:00:16 -0800 Subject: [PATCH 3/6] test(pipeline): xfail only the known second-interrupt defect, after the next row's work is done (ITL-1229) Review of #432 (CodeRabbit, Copilot): the strict xfail accepted any AssertionError, and the second press could come while a call of 'after' was still finishing. The test now has one row reaching 'after', presses the second interrupt only when that row's pool thread has finished its work item (back in its loop, or ended by the executor shutdown), keeps the other checks as plain assertions and raises a dedicated exception, the only one the xfail expects, for the known warning mismatch. Narrowing the xfail exposed that the helper waited forever for a pool thread that had already ended; it now counts an ended thread as done. Co-Authored-By: Claude Opus 5.5 --- .../test_job_run_in_event_loop.py | 50 +++++++++++++++---- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/tests/test_pipeline/test_job_run_in_event_loop.py b/tests/test_pipeline/test_job_run_in_event_loop.py index f870506de..a5759445f 100644 --- a/tests/test_pipeline/test_job_run_in_event_loop.py +++ b/tests/test_pipeline/test_job_run_in_event_loop.py @@ -10,6 +10,7 @@ import _thread import asyncio import signal +import sys import threading import time import warnings @@ -404,6 +405,7 @@ async def notebook_cell() -> None: run_ended = threading.Event() next_row_entered = threading.Event() next_row_returned = threading.Event() +next_row_threads: list[threading.Thread] = [] class RunEndSignal(NoOpObserver): @@ -415,15 +417,35 @@ def on_run_end(self, run_id: str) -> None: run_ended.set() +class NamedANodeWhoseFunctionReturned(Exception): + """The second-interrupt warning named a node whose function had returned (ITL-1192).""" + + +def pool_thread_is_done(thread: threading.Thread) -> bool: + """Whether the thread-pool ``thread`` has finished the work item it ran. + + It has once it is back in its loop waiting for work, or has ended (the cancelled + run shuts its thread pool down): the function and anything around it have returned. + """ + if not thread.is_alive() or thread.ident is None: + return True + frame = sys._current_frames().get(thread.ident) # pyright: ignore[reportPrivateUsage] # the standard way to read another thread's frame + return frame is None or frame.f_code.co_name == "_worker" + + def press_interrupt_once_next_row_returned() -> None: - """The second press, once the next node's function has returned.""" - if next_row_returned.wait(timeout=30): - _thread.interrupt_main() + """The second press, once the next node's function and its pool thread's work are done.""" + if not next_row_returned.wait(timeout=30): + return + deadline = time.monotonic() + 30 + while not pool_thread_is_done(next_row_threads[0]) and time.monotonic() < deadline: + time.sleep(0.01) + _thread.interrupt_main() @function_pod(output_keys="y") def stuck_until_next_row_runs(x: int) -> int: - """Row 0 presses the interrupt button once a row of the next node is in its function.""" + """Row 0 presses the interrupt button once the next node's row is in its function.""" if x == 0: _ = next_row_entered.wait(timeout=30) _thread.interrupt_main() # first press: cancels the run, which then waits @@ -435,6 +457,7 @@ def stuck_until_next_row_runs(x: int) -> int: @function_pod(output_keys="z") def returns_after_the_run_ended(y: int) -> int: """In progress when the cancellation lands; returns once the cancelled run has ended.""" + next_row_threads.append(threading.current_thread()) next_row_entered.set() _ = run_ended.wait(timeout=30) next_row_returned.set() @@ -443,7 +466,7 @@ def returns_after_the_run_ended(y: int) -> int: @pytest.mark.xfail( strict=True, - raises=AssertionError, + raises=NamedANodeWhoseFunctionReturned, reason=( "ITL-1192: a row task cancelled while its function runs gets no end event, so " "the warning names its node although the function has returned" @@ -453,15 +476,18 @@ def returns_after_the_run_ended(y: int) -> int: def test_second_interrupt_does_not_name_a_node_whose_function_returned() -> None: """The warning names only the nodes whose functions have not returned. - A row of ``after`` is in its function when the first interrupt cancels the run; the - function returns once the cancelled run has ended, and only then is the interrupt - pressed again, while row 0 of ``stuck`` still runs. Every step waits for the one + Two rows: row 0 stays in ``stuck``, row 1 goes on to ``after``. Row 1 of ``after`` is + in its function when the first interrupt cancels the run; the function returns once + the cancelled run has ended, and the interrupt is pressed again only when its pool + thread has finished that work, while row 0 of ``stuck`` still runs. Every step waits for the one before, so the outcome does not depend on timing. Today the warning names ``after`` - too: a known defect (ITL-1192), pinned here so the fix flips this test. + too: a known defect (ITL-1192), pinned here so the fix flips this test. Only that + defect is the expected failure; any other failed check fails the test. """ for event in (stuck_release, run_ended, next_row_entered, next_row_returned): event.clear() - rows = [{"k": f"r{i}", "x": i} for i in range(N_STUCK_ROWS)] + next_row_threads.clear() + rows = [{"k": f"r{i}", "x": i} for i in range(2)] job = PipelineJob(name="returned_demo", database=InMemoryArrowDatabase()) with job: source = DictSource( @@ -488,9 +514,11 @@ async def notebook_cell() -> None: time.sleep(0.05) assert next_row_returned.is_set() + assert len(next_row_threads) == 1, next_row_threads messages = [str(w.message) for w in caught if w.category is RuntimeWarning] assert len(messages) == 1, messages - assert "but a function in node 'stuck' has not returned yet" in messages[0] + if "but a function in node 'stuck' has not returned yet" not in messages[0]: + raise NamedANodeWhoseFunctionReturned(messages[0]) async def async_stuck_on_first_row(x: int) -> int: From e3fd4843bbdb0c85f30faac6d5a1d3f926388af1 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 17:20:56 -0800 Subject: [PATCH 4/6] test(pipeline): expect only the exact known second-interrupt warning (ITL-1229) Review of #432 (Copilot): any warning without the singular wording became the expected failure. Only the warning naming exactly 'stuck' and 'after' is now; any other wrong warning fails the plain assertion. Co-Authored-By: Claude Opus 5.5 --- tests/test_pipeline/test_job_run_in_event_loop.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_pipeline/test_job_run_in_event_loop.py b/tests/test_pipeline/test_job_run_in_event_loop.py index a5759445f..8d1c968ab 100644 --- a/tests/test_pipeline/test_job_run_in_event_loop.py +++ b/tests/test_pipeline/test_job_run_in_event_loop.py @@ -482,7 +482,8 @@ def test_second_interrupt_does_not_name_a_node_whose_function_returned() -> None thread has finished that work, while row 0 of ``stuck`` still runs. Every step waits for the one before, so the outcome does not depend on timing. Today the warning names ``after`` too: a known defect (ITL-1192), pinned here so the fix flips this test. Only that - defect is the expected failure; any other failed check fails the test. + warning, naming exactly ``stuck`` and ``after``, is the expected failure; any other + failed check, or any other wrong warning, fails the test. """ for event in (stuck_release, run_ended, next_row_entered, next_row_returned): event.clear() @@ -517,8 +518,10 @@ async def notebook_cell() -> None: assert len(next_row_threads) == 1, next_row_threads messages = [str(w.message) for w in caught if w.category is RuntimeWarning] assert len(messages) == 1, messages - if "but a function in node 'stuck' has not returned yet" not in messages[0]: + # The known defect, exactly; any other wrong warning fails the assertion below. + if "but a function in nodes 'stuck', 'after' has not returned yet" in messages[0]: raise NamedANodeWhoseFunctionReturned(messages[0]) + assert "but a function in node 'stuck' has not returned yet" in messages[0] async def async_stuck_on_first_row(x: int) -> int: From 525d57dee79e862188bd21e2bd8096a2c1c012a4 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 17:33:01 -0800 Subject: [PATCH 5/6] test(pipeline): second press only once the caller waits again; stop the helper in cleanup (ITL-1229) Review of #432 (Copilot): the helper now presses the second interrupt only when the main thread is back in the orchestrator's wait for the run (after the cancel, so the second wait), records whether the downstream function saw the run end and asserts it, and is stopped and joined in cleanup so it can never interrupt the cleanup or a later test. Co-Authored-By: Claude Opus 5.5 --- .../test_job_run_in_event_loop.py | 62 ++++++++++++++++--- 1 file changed, 54 insertions(+), 8 deletions(-) diff --git a/tests/test_pipeline/test_job_run_in_event_loop.py b/tests/test_pipeline/test_job_run_in_event_loop.py index 8d1c968ab..ac1acf5fd 100644 --- a/tests/test_pipeline/test_job_run_in_event_loop.py +++ b/tests/test_pipeline/test_job_run_in_event_loop.py @@ -406,6 +406,10 @@ async def notebook_cell() -> None: next_row_entered = threading.Event() next_row_returned = threading.Event() next_row_threads: list[threading.Thread] = [] +next_row_saw_run_end: list[bool] = [] +second_press_helpers: list[threading.Thread] = [] +second_press_stop = threading.Event() +second_press_lock = threading.Lock() class RunEndSignal(NoOpObserver): @@ -433,14 +437,40 @@ def pool_thread_is_done(thread: threading.Thread) -> bool: return frame is None or frame.f_code.co_name == "_worker" +def caller_waits_for_the_run() -> bool: + """Whether the main thread is blocked in the orchestrator's wait for the run's thread. + + Asked after the cancelled run has ended, so the wait is the one after the first + interrupt: the main thread cancels the run only after leaving the first wait. + """ + ident = threading.main_thread().ident + frame = sys._current_frames().get(ident) if ident is not None else None # pyright: ignore[reportPrivateUsage] # the standard way to read another thread's frame + while frame is not None: + code = frame.f_code + if code.co_name == "_wait_until" and code.co_filename.endswith( + "async_orchestrator.py" + ): + return True + frame = frame.f_back + return False + + def press_interrupt_once_next_row_returned() -> None: - """The second press, once the next node's function and its pool thread's work are done.""" + """The second press, once the next row's work is done and the caller waits again. + + Does not press once the test has stopped it (``second_press_stop``), so it never + interrupts the test's cleanup or a later test. + """ if not next_row_returned.wait(timeout=30): return deadline = time.monotonic() + 30 - while not pool_thread_is_done(next_row_threads[0]) and time.monotonic() < deadline: + while time.monotonic() < deadline and not second_press_stop.is_set(): + if pool_thread_is_done(next_row_threads[0]) and caller_waits_for_the_run(): + break time.sleep(0.01) - _thread.interrupt_main() + with second_press_lock: + if not second_press_stop.is_set(): + _thread.interrupt_main() @function_pod(output_keys="y") @@ -449,7 +479,9 @@ def stuck_until_next_row_runs(x: int) -> int: if x == 0: _ = next_row_entered.wait(timeout=30) _thread.interrupt_main() # first press: cancels the run, which then waits - threading.Thread(target=press_interrupt_once_next_row_returned).start() + helper = threading.Thread(target=press_interrupt_once_next_row_returned) + second_press_helpers.append(helper) + helper.start() _ = stuck_release.wait(timeout=30) return x @@ -459,7 +491,7 @@ def returns_after_the_run_ended(y: int) -> int: """In progress when the cancellation lands; returns once the cancelled run has ended.""" next_row_threads.append(threading.current_thread()) next_row_entered.set() - _ = run_ended.wait(timeout=30) + next_row_saw_run_end.append(run_ended.wait(timeout=30)) next_row_returned.set() return y + 1 @@ -479,15 +511,24 @@ def test_second_interrupt_does_not_name_a_node_whose_function_returned() -> None Two rows: row 0 stays in ``stuck``, row 1 goes on to ``after``. Row 1 of ``after`` is in its function when the first interrupt cancels the run; the function returns once the cancelled run has ended, and the interrupt is pressed again only when its pool - thread has finished that work, while row 0 of ``stuck`` still runs. Every step waits for the one - before, so the outcome does not depend on timing. Today the warning names ``after`` + thread has finished that work and the caller is back in its wait for the run, while + row 0 of ``stuck`` still runs. Every step waits for the one before, so the outcome + does not depend on timing. Today the warning names ``after`` too: a known defect (ITL-1192), pinned here so the fix flips this test. Only that warning, naming exactly ``stuck`` and ``after``, is the expected failure; any other failed check, or any other wrong warning, fails the test. """ - for event in (stuck_release, run_ended, next_row_entered, next_row_returned): + for event in ( + stuck_release, + run_ended, + next_row_entered, + next_row_returned, + second_press_stop, + ): event.clear() next_row_threads.clear() + next_row_saw_run_end.clear() + second_press_helpers.clear() rows = [{"k": f"r{i}", "x": i} for i in range(2)] job = PipelineJob(name="returned_demo", database=InMemoryArrowDatabase()) with job: @@ -508,6 +549,10 @@ async def notebook_cell() -> None: with pytest.raises(KeyboardInterrupt): loop.run_until_complete(notebook_cell()) finally: + with second_press_lock: + second_press_stop.set() + for helper in second_press_helpers: + helper.join(timeout=30) loop.close() stuck_release.set() deadline = time.monotonic() + 30 @@ -516,6 +561,7 @@ async def notebook_cell() -> None: assert next_row_returned.is_set() assert len(next_row_threads) == 1, next_row_threads + assert next_row_saw_run_end == [True], "the cancelled run did not end" messages = [str(w.message) for w in caught if w.category is RuntimeWarning] assert len(messages) == 1, messages # The known defect, exactly; any other wrong warning fails the assertion below. From 586244005aeef9637626c13a5cc0746a43470ade Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Tue, 17 Nov 2026 17:42:25 -0800 Subject: [PATCH 6/6] test(pipeline): no second press when the helper's wait times out (ITL-1229) Review of #432 (Copilot): on a synchronization timeout the helper returns instead of pressing, so the run ends without the warning and the test fails rather than reach the expected failure. Co-Authored-By: Claude Opus 5.5 --- tests/test_pipeline/test_job_run_in_event_loop.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/test_pipeline/test_job_run_in_event_loop.py b/tests/test_pipeline/test_job_run_in_event_loop.py index ac1acf5fd..3dad697b8 100644 --- a/tests/test_pipeline/test_job_run_in_event_loop.py +++ b/tests/test_pipeline/test_job_run_in_event_loop.py @@ -458,15 +458,17 @@ def caller_waits_for_the_run() -> bool: def press_interrupt_once_next_row_returned() -> None: """The second press, once the next row's work is done and the caller waits again. - Does not press once the test has stopped it (``second_press_stop``), so it never - interrupts the test's cleanup or a later test. + Does not press when that never happens (the test then fails rather than reach the + expected failure), nor once the test has stopped it (``second_press_stop``), so it + never interrupts the test's cleanup or a later test. """ if not next_row_returned.wait(timeout=30): return deadline = time.monotonic() + 30 - while time.monotonic() < deadline and not second_press_stop.is_set(): - if pool_thread_is_done(next_row_threads[0]) and caller_waits_for_the_run(): - break + while not (pool_thread_is_done(next_row_threads[0]) and caller_waits_for_the_run()): + if time.monotonic() > deadline or second_press_stop.is_set(): + # No press: the run then ends without the warning, and the test fails. + return time.sleep(0.01) with second_press_lock: if not second_press_stop.is_set():