Ignore runner callback HTTP and JSON blips in the curation breaker

backend/services/codex_add_evals_service.py

@@ -221,11 +221,16 @@ def _is_callback_validation_rejection(row: PaperCurationJob) -> bool:
     return (getattr(row, "validation_summary", None) or {}).get("status") == "rejected"
 
 
-# The runner's generic failure for a callback that never got a response: the
-# HF Job could not reach the site, which says nothing about the curation itself.
+# The runner's generic failure for one of its own callbacks to the site
+# (bootstrap, context, complete), which says nothing about the curation itself.
+# An apply runs inside Codex, whose rejections never surface under these names.
+# HTTPError and JSONDecodeError are almost always the edge answering for us (a
+# Cloudflare 5xx or an empty or HTML body). A systemic rejection, such as a
+# release that cannot serve its bootstrap bundle, still trips the transport
+# breaker below.
 _TRANSPORT_FAILURE = re.compile(
     r"^Remote runner failed \((URLError|RemoteDisconnected|TimeoutError|"
-    r"ConnectionResetError|IncompleteRead)\)\.$"
+    r"ConnectionResetError|IncompleteRead|HTTPError|JSONDecodeError)\)\.$"
 )
 
 

backend/tests/curation/test_completion_atomicity.py

@@ -347,13 +347,35 @@ def test_runner_exception_names_outside_transport_still_count(completion):
         db,
         job,
         report="",
-        error="Remote runner failed (JSONDecodeError).",
+        error="Remote runner failed (CalledProcessError).",
         settings=settings,
     )
 
     assert state.consecutive_apply_failures == 1
 
 
+def test_runner_callback_blips_and_awaiting_markdown_do_not_pause(completion):
+    # The rows that paused production dispatch on 2026-11-03 at 04:31 UTC.
+    db, service, settings, job = completion
+    job.parent_workflow_id = None
+    state = db.get(PaperCurationSchedulerState, 1)
+    state.consecutive_apply_failures = 0
+    _add_history(db, job, [None] * 24)
+    errors = [
+        "Remote runner failed (JSONDecodeError).",
+        "Remote runner failed (HTTPError).",
+        "[awaiting_markdown] Awaiting paper Markdown extraction: 2511.00001",
+        "Remote runner failed (JSONDecodeError).",
+    ]
+    for error in errors:
+        job.status = "running"
+        db.commit()
+        service.complete(db, job, report="", error=error, settings=settings)
+
+    assert state.consecutive_apply_failures == 0
+    assert state.paused_at is None
+
+
 def test_last_canary_completion_releases_backlog_without_autoflush(completion):
     db, service, settings, job = completion
     job.kind = "add-evals"

docs/operations.md

@@ -761,9 +761,11 @@ age and becomes critical at five hours. Scheduled dispatch automatically pauses
 after a server invariant breach, three consecutive trace/apply/verification
 failures, a greater-than-20% failure rate in the latest 25 stages, or the
 post-baseline daily token guardrail. Neither failure counter includes
-source-evidence failures or runner transport failures (`Remote runner failed
-(URLError|RemoteDisconnected|TimeoutError|…)`, a callback that never got a
-response), since neither says anything about the curation; more than half of the latest 25
+source-evidence failures (including `[awaiting_markdown]`) or runner transport
+failures (`Remote runner failed
+(URLError|RemoteDisconnected|TimeoutError|HTTPError|JSONDecodeError|…)`, one of
+the runner's own callbacks failing or getting an edge error page instead of
+JSON), since neither says anything about the curation; more than half of the latest 25
 stages failing that way pauses as `runner_transport_failures` instead. Resuming is an explicit acknowledgement
 that resets the consecutive counters and starts a fresh 25-stage failure-rate
 window; a recurring invariant breach still pauses immediately.