feat(proxy): fail over on throttle-403 with short cooldown

src/proxy/quota-detector.ts

@@ -37,6 +37,38 @@ export function isQuota429(statusCode: number, headers: Record<string, string>,
   }
 }
 
+// NOTE: mirrored by isTransientThrottleMessage in openclaude
+// src/services/api/openaiErrorClassification.ts — keep the two in sync.
+export function isThrottle403(statusCode: number, body: string): boolean {
+  if (statusCode !== 403) return false
+  const lower = body.toLowerCase()
+  if (
+    lower.includes('quota') ||
+    lower.includes('exhausted') ||
+    lower.includes('usage limit') ||
+    lower.includes('usage_not_included') ||
+    lower.includes('freeusagelimit') ||
+    lower.includes('credit balance') ||
+    lower.includes('billing limit') ||
+    lower.includes('insufficient_quota') ||
+    lower.includes('payment required') ||
+    lower.includes('allotment') ||
+    lower.includes('limit: 0')
+  )
+    return false // billing stays terminal
+  if (
+    /unauthorized|invalid.{0,10}(key|token)|api key|forbidden|access denied|revoked|expired|permission|not allowed|org_|insufficient_quota|auth/.test(
+      lower,
+    )
+  )
+    return false
+  const trimmed = body.trim()
+  if (trimmed === '') return true // the Zen volume-gating shape
+  return /rate.?limit|too many requests|try again|slow down|throttl|overloaded|capacity|busy/.test(
+    lower,
+  )
+}
+
 export function parseRetryAfterHeaderMs(headers: Record<string, string>, now: number): number | null {
   const retryAfterMs = headers['retry-after-ms']
   if (retryAfterMs) {

src/proxy/server.ts

@@ -18,7 +18,7 @@ import {
 import { buildUpstreamHeaders, extractCacheHeaders } from './header-passthrough.js'
 import { applyCodeafCompat, type HarnessStamp } from './codeaf-compat.js'
 import { isChatCompletionsPath, toResponsesPath, toResponsesRequestBody, toChatCompletion, SseTranslator, type TranslatorStats } from './zen-responses.js'
-import { isLocalNetworkOutage, isQuota429, parseRetryAfterHeaderMs, resolveCooldownMs } from './quota-detector.js'
+import { isLocalNetworkOutage, isQuota429, isThrottle403, parseRetryAfterHeaderMs, resolveCooldownMs } from './quota-detector.js'
 import { parseUsageData } from './response-parser.js'
 import { estimateCost } from './rate-card.js'
 import { SessionAffinityStore } from './session-affinity.js'
@@ -501,6 +501,54 @@ export class ProxyServer {
           }
         }
 
+        if (upstreamRes.status === 403) {
+          const throttleBody = await responseTextPromise
+          if (isThrottle403(upstreamRes.status, throttleBody)) {
+            const throttleCooldownMs = resolveCooldownMs(Object.fromEntries(upstreamRes.headers), throttleBody, Date.now(), 60_000)
+            // NOTE: no QuotaErrorSignal — markExhausted with a signal increments
+            // quotaErrorCount (key-manager.ts:177-186) and would pollute quota
+            // dashboards + notifier.keyExhausted. Throttle is not quota.
+            this.keyManager.markExhausted(key.id, throttleCooldownMs)
+            this.circuitBreaker.recordFailure(key.id)
+            this.keyManager.markError(key.id)
+            if (attempt < maxAttempts - 1) {
+              // Counted here; on the last attempt the tail recordRequest
+              // below counts the verbatim 403 once (no double-count).
+              this.keyManager.recordRequest(key.id, {
+                statusCode: upstreamRes.status,
+                durationMs: duration,
+                model: prepared.model,
+                sessionId: upstreamSessionId ?? sessionKey ?? null,
+                successful: false,
+              })
+            }
+            attemptedKeyIds.add(key.id)
+            this.logStream.emit(
+              this.logger,
+              'warn',
+              `Key "${key.alias}" throttled (HTTP 403), cooldown ${throttleCooldownMs}ms, failing over`,
+              {
+                method: req.method,
+                harness, compatApplied, clientUa,
+                path: targetPath,
+                statusCode: upstreamRes.status,
+                keyAlias: key.alias,
+                keyId: key.id,
+                duration,
+                model: prepared.model,
+                strategy,
+                routeReason: reason,
+                selectedBySession,
+                sessionId: upstreamSessionId ?? sessionKey ?? null,
+                cooldownMs: throttleCooldownMs,
+                attempt: attempt + 1,
+                upstream: isZenRequest ? 'zen' : 'go',
+              },
+            )
+            if (attempt < maxAttempts - 1) continue
+          }
+        } // else: existing count-only path below, unchanged
+
         if (upstreamRes.status >= 500) {
           const repeatedServerError = upstreamServerErrorSeen
           upstreamServerErrorSeen = true