From 0a345afc6201ec64e97ca0e1dd0ef47c568d75a7 Mon Sep 17 00:00:00 2001 From: "Edgar Y. Walker" <> Date: Thu, 12 Nov 2026 21:57:27 -0800 Subject: [PATCH] test(crash_recovery): crash mid-batch only after the earlier batches commit (ITL-1001) test_crash_mid_batch_keeps_committed_batches[async] failed now and then on CI with records == 100 instead of 200. The async orchestrator computes rows on a thread pool ahead of the stage that records them and commits every 100 rows: when the second commit starts, the pool has started rows up to 223 to 239. Row 234 could therefore start, and exit the process, before the second batch's pipeline records were committed. Nothing is lost (the first batch is intact); the test assumed an ordering the library does not promise. The job script now makes the crash row, and every row after it, wait on an event that the store sets once its flushes for the batches before that row have returned; only then does the crash row exit. The crash falls in the middle of the third batch on both paths, the result bound 200 <= results < 234 holds deterministically, and a missing commit exits with code 4, which run_job rejects, so the test fails instead of hanging. Crashing at row 210 (inside the measured lag) shows the difference: the old form stored 100 records in 40 of 40 runs, the new form 200 in 40 of 40. Co-Authored-By: Claude Opus 5.5 --- .../test_pipeline/test_job_crash_recovery.py | 39 ++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/tests/test_pipeline/test_job_crash_recovery.py b/tests/test_pipeline/test_job_crash_recovery.py index 1d10aaa30..188a9af17 100644 --- a/tests/test_pipeline/test_job_crash_recovery.py +++ b/tests/test_pipeline/test_job_crash_recovery.py @@ -31,13 +31,25 @@ # Runs one job over N_ROWS rows. Arguments: store path, calls file, commit_every_rows, # orchestrator ("sync" or "async"), crash ("none", "fn:" or "flush:"). -# "fn:" exits while computing row x; "flush:" exits on the n-th store-wide -# flush. The run's first flush commits what was pending before it; each later one is -# a node's commit, called right after the node committed its result table. +# "fn:" exits while computing row x, once the batches of the rows before x are +# committed; "flush:" exits on the n-th store-wide flush. The run's first flush +# commits what was pending before it; each later one is a node's commit, called right +# after the node committed its result table. +# +# The async orchestrator computes rows on a thread pool ahead of the stage that records +# them and commits every commit_every_rows rows, so without a wait row x could be +# computed before the batches before it are committed (ITL-1001). Row x and every row +# after it wait until the store has finished the run's first flush and one flush per +# batch before x, so the crash falls in the batch that holds x, after the earlier +# batches are committed and before the result of any row from x on is stored. On the +# sync path the rows come in order and the wait returns at once. The wait is bounded: if +# the commits never come, the job exits with code 4, which run_job rejects, so the test +# fails instead of hanging. JOB_SCRIPT = textwrap.dedent( """ import os import sys + import threading from typing import Any from orcapod import PipelineJob, function_pod @@ -47,14 +59,29 @@ store_path, calls_path, every, orchestrator, crash = sys.argv[1:6] crash_kind, _, crash_arg = crash.partition(":") + earlier_batches_committed = threading.Event() + if crash_kind == "fn": + flushes_before_crash = 1 + int(crash_arg) // int(every) + + + def wait_for_earlier_batches() -> None: + if not earlier_batches_committed.wait(timeout=120): + print( + "the batches before the crash row were never committed", + file=sys.stderr, + flush=True, + ) + os._exit(4) @function_pod(output_keys="y") def double(x: int) -> int: with open(calls_path, "a") as calls: calls.write(f"{x}\\n") - if crash_kind == "fn" and x == int(crash_arg): - os._exit(3) + if crash_kind == "fn" and x >= int(crash_arg): + wait_for_earlier_batches() + if x == int(crash_arg): + os._exit(3) return 2 * x @@ -71,6 +98,8 @@ def flush(self) -> None: if crash_kind == "flush" and self._state["n"] == int(crash_arg): os._exit(3) self._inner.flush() + if crash_kind == "fn" and self._state["n"] >= flushes_before_crash: + earlier_batches_committed.set() def __getattr__(self, name: str) -> Any: return getattr(self._inner, name)