fix(heartbeat,adapter-k8s): a 429 behind a replacement Job is the throttle, not k8s_pod_schedule_failed (BLO-34577)

server/src/__tests__/heartbeat-rate-limit-exhausted.test.ts

@@ -13,8 +13,10 @@ import {
   countConsecutiveZeroTokenCompletedRuns,
   isRateLimitExhausted,
   isRetryableK8sCcrotateThrottleResult,
+  K8S_REPLACEMENT_LAUNCH_FAILURE_AFTER_THROTTLE_KEY,
   k8sCcrotateRetryDelayMs,
   listRecentTerminalRunsForZeroTokenStreak,
+  reclassifyK8sReplacementLaunchFailureAfterThrottle,
 } from "../services/heartbeat.js";
 import {
   getEmbeddedPostgresTestSupport,
@@ -401,6 +403,118 @@ describe("k8s ccrotate no-progress throttle detection", () => {
   });
 });
 
+// BLO-34577: the in-run throttle loop relaunches the Job for the same run. On
+// 2026-09-18 the relaunch read the previous attempt's Failed pod and returned
+// `k8s_pod_schedule_failed` ~100 ms after create; the finalizer recorded that
+// code and the pr_review run was dropped without a retry, while the identical
+// 429 seen through the throttle path retried. These pin the server-side verdict.
+describe("reclassifyK8sReplacementLaunchFailureAfterThrottle (BLO-34577)", () => {
+  const TENANT_429 =
+    'API Error: 429 {"type":"error","error":{"type":"rate_limit_error","message":"All Claude subscription capacity for this tenant is rate-limited"}}';
+  const zeroUsage = { inputTokens: 0, outputTokens: 0, cachedInputTokens: 0 };
+  const throttleResult = {
+    exitCode: 1,
+    signal: null,
+    timedOut: false,
+    errorMessage: TENANT_429,
+    errorCode: null,
+    retryNotBefore: "2026-09-18T12:40:00.000Z",
+    resultJson: { api_error_status: 429, is_error: true },
+    usage: zeroUsage,
+  };
+  const launchFailure = {
+    exitCode: null,
+    signal: null,
+    timedOut: false,
+    errorMessage:
+      "Pod scheduling failed: Pod ac-ally-96fa0c75-3f2a1b-x9k2q reached phase=Failed: claude exited 1",
+    errorCode: "k8s_pod_schedule_failed",
+  };
+
+  it("finalizes a replacement-launch failure after an in-run throttle with the throttle verdict", () => {
+    const reclassified = reclassifyK8sReplacementLaunchFailureAfterThrottle({
+      launchResult: launchFailure,
+      throttleResult,
+      throttleAttempts: 2,
+    });
+    expect(reclassified).not.toBeNull();
+    // The verdict is the throttle's: the code the finalizer treats as terminal is gone...
+    expect(reclassified!.errorCode).not.toBe("k8s_pod_schedule_failed");
+    // ...and the result still classifies as the in-run throttle the loop was retrying,
+    // so the finalizer takes the provider_throttled_no_progress / rate_limit_exhausted arm.
+    expect(isRetryableK8sCcrotateThrottleResult(reclassified!)).toBe(true);
+    expect(reclassified!.resultJson).toMatchObject({ api_error_status: 429, is_error: true });
+    expect(reclassified!.retryNotBefore).toBe("2026-09-18T12:40:00.000Z");
+    // The launch failure is kept as an annotation, not lost.
+    expect(reclassified!.resultJson?.[K8S_REPLACEMENT_LAUNCH_FAILURE_AFTER_THROTTLE_KEY]).toEqual({
+      errorCode: "k8s_pod_schedule_failed",
+      errorMessage: launchFailure.errorMessage,
+      throttleAttempts: 2,
+      throttleErrorCode: null,
+    });
+    expect(reclassified!.errorMessage).toContain("rate-limited");
+    expect(reclassified!.errorMessage).toContain("after 2 in-run throttle retries");
+    expect(reclassified!.errorMessage).toContain("phase=Failed: claude exited 1");
+  });
+
+  it("leaves an ambiguous k8s_pod_schedule_failed alone when no throttle preceded it", () => {
+    // Negative control: with no observed throttle the launch failure means what it
+    // says, and the existing "does not retry ambiguous k8s_pod_schedule_failed"
+    // contract must keep applying to it.
+    expect(
+      reclassifyK8sReplacementLaunchFailureAfterThrottle({
+        launchResult: launchFailure,
+        throttleResult: null,
+        throttleAttempts: 0,
+      }),
+    ).toBeNull();
+    expect(
+      reclassifyK8sReplacementLaunchFailureAfterThrottle({
+        launchResult: launchFailure,
+        throttleResult,
+        throttleAttempts: 0,
+      }),
+    ).toBeNull();
+  });
+
+  it("only reclassifies a launch failure, never another terminal result", () => {
+    for (const launchResult of [
+      { exitCode: 0, signal: null, timedOut: false, usage: { inputTokens: 40, outputTokens: 12 } },
+      { exitCode: 1, signal: null, timedOut: false, errorCode: "adapter_failed", errorMessage: "boom" },
+      { exitCode: null, signal: null, timedOut: false, errorCode: "k8s_concurrent_run_blocked" },
+    ]) {
+      expect(
+        reclassifyK8sReplacementLaunchFailureAfterThrottle({
+          launchResult,
+          throttleResult,
+          throttleAttempts: 1,
+        }),
+      ).toBeNull();
+    }
+  });
+
+  it("does not let a non-throttle prior result stand in as the verdict", () => {
+    // The loop never retries a result with token usage, so a prior result that
+    // made progress is not a throttle chain; do not manufacture one.
+    expect(
+      reclassifyK8sReplacementLaunchFailureAfterThrottle({
+        launchResult: launchFailure,
+        throttleResult: { ...throttleResult, usage: { inputTokens: 12, outputTokens: 3 } },
+        throttleAttempts: 1,
+      }),
+    ).toBeNull();
+    // Nor a launch failure that somehow reports usage: the pod is claimed to
+    // have never run, so this is not the shape the reclassifier understands.
+    expect(
+      reclassifyK8sReplacementLaunchFailureAfterThrottle({
+        launchResult: { ...launchFailure, usage: { inputTokens: 1, outputTokens: 0 } },
+        throttleResult,
+        throttleAttempts: 1,
+      }),
+    ).toBeNull();
+  });
+});
+
 describe("countConsecutiveZeroTokenCompletedRuns", () => {
   it("counts only the newest terminal zero-token prefix", () => {
     expect(countConsecutiveZeroTokenCompletedRuns([

server/src/__tests__/heartbeat-retry-scheduling.test.ts

@@ -33,6 +33,7 @@ import {
   INTERACTION_CONTINUATION_INFRA_RETRY_REASON,
   INTERACTION_CONTINUATION_INFRA_WAKE_REASON,
   JOB_FAILED_HEARTBEAT_RETRY_MAX_ATTEMPTS,
+  K8S_REPLACEMENT_LAUNCH_FAILURE_AFTER_THROTTLE_KEY,
   MAX_TURN_CONTINUATION_RETRY_REASON,
   MAX_TURN_CONTINUATION_WAKE_REASON,
   heartbeatService,
@@ -3245,6 +3246,51 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => {
     }
   });
 
+  // BLO-34577: a replacement Job launch that failed before its pod ran, after
+  // this run already observed a zero-progress 429, is finalized with the throttle
+  // verdict (provider_throttled_no_progress / rate_limit_exhausted) carrying the
+  // launch failure as an annotation. That verdict is what re-queues the pr_review
+  // -- previously the run kept `k8s_pod_schedule_failed` and the review dropped.
+  describe("BLO-34577 replacement-launch failure after an in-run throttle", () => {
+    const annotation = {
+      [K8S_REPLACEMENT_LAUNCH_FAILURE_AFTER_THROTTLE_KEY]: {
+        errorCode: "k8s_pod_schedule_failed",
+        errorMessage: "Pod scheduling failed: Pod ac-ally-x reached phase=Failed: claude exited 1",
+        throttleAttempts: 2,
+        throttleErrorCode: null,
+      },
+    };
+
+    it("schedules the bounded retry for pr_review and issue contexts once finalized as the throttle", () => {
+      for (const contextSnapshot of [
+        { wakeReason: "github_pr_opened", reviewKind: "pr_review", githubPrNumber: 3212 },
+        { taskKey: "pr_review:Blockcast/pim-multicast-gateway:3212" },
+        { issueId: randomUUID(), wakeReason: "issue_assigned" },
+      ]) {
+        expect(
+          shouldScheduleAutomaticRunRetry({
+            errorCode: "provider_throttled_no_progress",
+            resultJson: { errorFamily: "rate_limit_exhausted", api_error_status: 429, ...annotation },
+            contextSnapshot,
+          }),
+        ).toBe(true);
+      }
+    });
+
+    it("does not retry when the launch failure was recorded verbatim, annotation or not", () => {
+      // Negative control: the fix is the VERDICT, not the annotation. A run that
+      // kept `k8s_pod_schedule_failed` -- and a stale transient family in its
+      // merged resultJson -- still hits the ambiguous-outcome reject first.
+      expect(
+        shouldScheduleAutomaticRunRetry({
+          errorCode: "k8s_pod_schedule_failed",
+          resultJson: { errorFamily: "rate_limit_exhausted", ...annotation },
+          contextSnapshot: { wakeReason: "github_pr_opened", reviewKind: "pr_review", githubPrNumber: 3212 },
+        }),
+      ).toBe(false);
+    });
+  });
+
   // BLO-17456: when a PR-review chain exhausts, the reviewer never posts its
   // required status, so the PR sits on "Expected — waiting for status" forever.
   // These drive the real exhaustion path (no mocks): loadConfig() reads

server/src/services/heartbeat.ts

@@ -5422,6 +5422,70 @@ export function k8sCcrotateRetryDelayMs(result: { retryNotBefore?: string | null
   );
 }
 
+// BLO-34577: the in-run ccrotate throttle loop re-invokes the k8s adapter for
+// the SAME runId after a zero-progress 429. Each replacement attempt is a fresh
+// Job launch, and that launch can itself fail before any pod runs
+// (`k8s_pod_schedule_failed`). Observed 2026-09-18 on 7 Ally pr_review runs:
+// the adapter read the PREVIOUS attempt's Failed pod -- same deterministic Job
+// name, deleted in the background -- as the replacement's ~100 ms after create
+// and returned `k8s_pod_schedule_failed`. The loop broke on that non-throttle
+// result and the finalizer recorded the code verbatim, which
+// `shouldScheduleAutomaticRunRetry` and `isNonRetryablePrReviewTerminalOutcome`
+// treat as terminal: no retry was minted, the review was dropped, and the gate
+// posted `non_retryable_external_lifecycle`. The identical 429 finalized through
+// the throttle path retries on the flat rate-limit curve.
+//
+// The adapter no longer reads a foreign pod (it scopes lookups to the created
+// Job's UID), but the server verdict must not depend on that: once this run has
+// observed >= 1 zero-progress throttle, a replacement launch that fails before
+// its pod runs adds no information about the WORK -- no attempt made model
+// progress (the loop only retries zero-token results) and this attempt's pod
+// never ran -- so the run's cause is still the throttle. Finalize with the
+// throttle verdict and record the launch failure as an annotation, so the
+// `provider_throttled_no_progress` / `rate_limit_exhausted` path schedules the
+// bounded retry it would have scheduled had the loop simply exhausted.
+//
+// Deliberately narrow: a `k8s_pod_schedule_failed` with NO prior throttle in
+// this run is left exactly as reported. That outcome is ambiguous, and the
+// "does not retry ambiguous k8s_pod_schedule_failed" contract still holds.
+export const K8S_REPLACEMENT_LAUNCH_FAILURE_AFTER_THROTTLE_KEY =
+  "replacementLaunchFailureAfterThrottle" as const;
+
+export function reclassifyK8sReplacementLaunchFailureAfterThrottle(input: {
+  launchResult: AdapterExecutionResult;
+  throttleResult: AdapterExecutionResult | null;
+  throttleAttempts: number;
+}): AdapterExecutionResult | null {
+  const { launchResult, throttleResult, throttleAttempts } = input;
+  if (!throttleResult || throttleAttempts < 1) return null;
+  if (launchResult.errorCode !== "k8s_pod_schedule_failed") return null;
+  // A pod that never ran cannot have made model progress. If usage is reported
+  // anyway this is not the shape described above; leave the verdict alone.
+  if (!zeroTokenUsage(launchResult.usage)) return null;
+  // Only a result the loop itself judged a retryable throttle may stand in as
+  // the verdict; anything else and this was not a throttle chain.
+  if (!isRetryableK8sCcrotateThrottleResult(throttleResult)) return null;
+  const launchErrorMessage = readNonEmptyString(launchResult.errorMessage) ?? launchResult.errorCode;
+  const throttleErrorMessage =
+    readNonEmptyString(throttleResult.errorMessage) ?? "Provider throttled before model progress";
+  const retryNoun = throttleAttempts === 1 ? "retry" : "retries";
+  return {
+    ...throttleResult,
+    errorMessage:
+      `${throttleErrorMessage} (replacement Job launch after ${throttleAttempts} in-run throttle ` +
+      `${retryNoun} failed before its pod ran: ${launchErrorMessage})`,
+    resultJson: {
+      ...(throttleResult.resultJson ?? {}),
+      [K8S_REPLACEMENT_LAUNCH_FAILURE_AFTER_THROTTLE_KEY]: {
+        errorCode: launchResult.errorCode,
+        errorMessage: launchResult.errorMessage ?? null,
+        throttleAttempts,
+        throttleErrorCode: throttleResult.errorCode ?? null,
+      },
+    },
+  };
+}
+
 // The pinned claude_k8s/opencode_k8s adapters report their launch command as
 // this exact "kubectl job/<name>" sentinel (not the real invoked command) so
 // the reservation can learn the expected Job name before the post-create
@@ -31155,6 +31219,10 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
               }
             }
             let ccrotateRetryAttempt = 0;
+            // BLO-34577: the most recent result the loop judged a retryable
+            // throttle. A replacement launch that fails before its pod runs is
+            // finalized with THIS verdict, not the launch failure's.
+            let lastInRunThrottleResult: Awaited<ReturnType<typeof adapter.execute>> | null = null;
             while (true) {
               const executionReservation = externalRuntimeReservation;
               if (executionReservation) {
@@ -31240,6 +31308,39 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
                 !isRetryableK8sCcrotateThrottleResult(adapterResult) ||
                 ccrotateRetryAttempt >= K8S_CCROTATE_IN_RUN_RETRY_MAX_ATTEMPTS
               ) {
+                // BLO-34577: a replacement Job whose launch failed before its
+                // pod ran, after this run already observed a zero-progress
+                // throttle, is still the throttle -- see
+                // reclassifyK8sReplacementLaunchFailureAfterThrottle. Without
+                // this the run finalized as `k8s_pod_schedule_failed`, which is
+                // terminal for pr_review, and the review was silently dropped.
+                const reclassified = isK8sAdapter(agent.adapterType)
+                  ? reclassifyK8sReplacementLaunchFailureAfterThrottle({
+                      launchResult: adapterResult,
+                      throttleResult: lastInRunThrottleResult,
+                      throttleAttempts: ccrotateRetryAttempt,
+                    })
+                  : null;
+                if (reclassified) {
+                  await appendRunEvent(currentRun, seq++, {
+                    eventType: "lifecycle",
+                    stream: "system",
+                    level: "warn",
+                    message:
+                      "replacement Job launch failed after an in-run ccrotate throttle; finalizing with the throttle verdict so the bounded retry is scheduled",
+                    payload: {
+                      launchErrorCode: adapterResult.errorCode ?? null,
+                      launchErrorMessage: adapterResult.errorMessage ?? null,
+                      throttleAttempts: ccrotateRetryAttempt,
+                      maxAttempts: K8S_CCROTATE_IN_RUN_RETRY_MAX_ATTEMPTS,
+                    },
+                  });
+                  await onLog(
+                    "stderr",
+                    `[paperclip] Replacement Job launch failed after ${ccrotateRetryAttempt} in-run throttle ${ccrotateRetryAttempt === 1 ? "retry" : "retries"}; recording the run as provider-throttled so it is retried.\n`,
+                  );
+                  adapterResult = reclassified;
+                }
                 break;
               }
               // BLO-18278: if the provider advertised a reset the in-run loop
@@ -31294,6 +31395,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
                 break;
               }
               ccrotateRetryAttempt += 1;
+              lastInRunThrottleResult = adapterResult;
               const retryDelayMs = k8sCcrotateRetryDelayMs(adapterResult);
               if (externalRuntimeReservation) {
                 // The completed Job belongs to the attempt that just returned.

vendor/paperclip-adapter-claude-k8s/PROVENANCE.md

@@ -14,7 +14,7 @@ control plane.
 | Repository vendored from | <https://github.com/kkroo/paperclip-adapter-claude-k8s> |
 | Package | `paperclip-adapter-claude-k8s` |
 | Version at vendor time | `0.2.5-kkroo.6` |
-| Current version | `0.2.6-blockcast.10` — see [Versioning](#versioning) |
+| Current version | `0.2.6-blockcast.11` — see [Versioning](#versioning) |
 | Declared license | MIT, in `package.json` only — see the caveat below |
 
 Before this change the image built this package by cloning that repository at a
@@ -96,7 +96,7 @@ A manifest of `sha256(path)` over all 41 in-tree files, sorted by path under
 `LC_ALL=C`, itself hashes to:
 
 ```
-7a91abbddc0522fe48ba8480d69be634600217c065341481efc1c1bb3cece4d3
+8c3a5f3d741ff0567bbe9f4a33ff7e9b520396dbc3cebe635d12d23b5f81fdf4
 ```
 
 Regenerate with:
@@ -193,7 +193,7 @@ after the first Blockcast change that ships, the version alone could no longer
 tell you which code was running — provenance had to be established by grepping
 `dist/` for a token.
 
-This directory therefore versions itself: **`0.2.6-blockcast.10`**, set in
+This directory therefore versions itself: **`0.2.6-blockcast.11`**, set in
 `package.json` and `package-lock.json`. The `-blockcast.` prerelease channel
 says plainly that this is our tree, not an upstream release.
 

vendor/paperclip-adapter-claude-k8s/package.json

@@ -1,6 +1,6 @@
 {
   "name": "paperclip-adapter-claude-k8s",
-  "version": "0.2.6-blockcast.10",
+  "version": "0.2.6-blockcast.11",
   "description": "Paperclip adapter plugin that runs Claude Code agents as Kubernetes Jobs",
   "license": "MIT",
   "repository": {