diff --git a/backend/services/codex_add_evals_service.py b/backend/services/codex_add_evals_service.py index 2494fcde..1f79496b 100644 --- a/backend/services/codex_add_evals_service.py +++ b/backend/services/codex_add_evals_service.py @@ -221,11 +221,16 @@ def _is_callback_validation_rejection(row: PaperCurationJob) -> bool: return (getattr(row, "validation_summary", None) or {}).get("status") == "rejected" -# The runner's generic failure for a callback that never got a response: the -# HF Job could not reach the site, which says nothing about the curation itself. +# The runner's generic failure for one of its own callbacks to the site +# (bootstrap, context, complete), which says nothing about the curation itself. +# An apply runs inside Codex, whose rejections never surface under these names. +# HTTPError and JSONDecodeError are almost always the edge answering for us (a +# Cloudflare 5xx or an empty or HTML body). A systemic rejection, such as a +# release that cannot serve its bootstrap bundle, still trips the transport +# breaker below. _TRANSPORT_FAILURE = re.compile( r"^Remote runner failed \((URLError|RemoteDisconnected|TimeoutError|" - r"ConnectionResetError|IncompleteRead)\)\.$" + r"ConnectionResetError|IncompleteRead|HTTPError|JSONDecodeError)\)\.$" ) diff --git a/backend/tests/curation/test_completion_atomicity.py b/backend/tests/curation/test_completion_atomicity.py index a45e0c3c..df3ede3f 100644 --- a/backend/tests/curation/test_completion_atomicity.py +++ b/backend/tests/curation/test_completion_atomicity.py @@ -347,13 +347,35 @@ def test_runner_exception_names_outside_transport_still_count(completion): db, job, report="", - error="Remote runner failed (JSONDecodeError).", + error="Remote runner failed (CalledProcessError).", settings=settings, ) assert state.consecutive_apply_failures == 1 +def test_runner_callback_blips_and_awaiting_markdown_do_not_pause(completion): + # The rows that paused production dispatch on 2026-11-03 at 04:31 UTC. + db, service, settings, job = completion + job.parent_workflow_id = None + state = db.get(PaperCurationSchedulerState, 1) + state.consecutive_apply_failures = 0 + _add_history(db, job, [None] * 24) + errors = [ + "Remote runner failed (JSONDecodeError).", + "Remote runner failed (HTTPError).", + "[awaiting_markdown] Awaiting paper Markdown extraction: 2511.00001", + "Remote runner failed (JSONDecodeError).", + ] + for error in errors: + job.status = "running" + db.commit() + service.complete(db, job, report="", error=error, settings=settings) + + assert state.consecutive_apply_failures == 0 + assert state.paused_at is None + + def test_last_canary_completion_releases_backlog_without_autoflush(completion): db, service, settings, job = completion job.kind = "add-evals" diff --git a/docs/operations.md b/docs/operations.md index c005a595..7dc7a14e 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -761,9 +761,11 @@ age and becomes critical at five hours. Scheduled dispatch automatically pauses after a server invariant breach, three consecutive trace/apply/verification failures, a greater-than-20% failure rate in the latest 25 stages, or the post-baseline daily token guardrail. Neither failure counter includes -source-evidence failures or runner transport failures (`Remote runner failed -(URLError|RemoteDisconnected|TimeoutError|…)`, a callback that never got a -response), since neither says anything about the curation; more than half of the latest 25 +source-evidence failures (including `[awaiting_markdown]`) or runner transport +failures (`Remote runner failed +(URLError|RemoteDisconnected|TimeoutError|HTTPError|JSONDecodeError|…)`, one of +the runner's own callbacks failing or getting an edge error page instead of +JSON), since neither says anything about the curation; more than half of the latest 25 stages failing that way pauses as `runner_transport_failures` instead. Resuming is an explicit acknowledgement that resets the consecutive counters and starts a fresh 25-stage failure-rate window; a recurring invariant breach still pauses immediately.