From 1a7dcb787d4be81cb2559c65f2f4ec5c7063ed2f Mon Sep 17 00:00:00 2001 From: Reinis Date: Sat, 10 Oct 2026 13:02:07 +0300 Subject: [PATCH 1/2] feat(proxy): fail over on throttle-403 with short cooldown Empty-body / throttle-phrased upstream 403 cools the hot key (60s fallback via resolveCooldownMs, no quota signal) and fails over same-request. Genuine auth 403s stay count-only. Plan: docs/opencode-go-multi-auth/plans/2026-10-10-throttle-403-failover.md. Mirror of openclaude 9f9c4e5 isTransientThrottleMessage. --- src/proxy/quota-detector.ts | 32 ++++++++++++++++++++++++++ src/proxy/server.ts | 46 ++++++++++++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/src/proxy/quota-detector.ts b/src/proxy/quota-detector.ts index d0a251d..7767e64 100644 --- a/src/proxy/quota-detector.ts +++ b/src/proxy/quota-detector.ts @@ -37,6 +37,38 @@ export function isQuota429(statusCode: number, headers: Record, } } +// NOTE: mirrored by isTransientThrottleMessage in openclaude +// src/services/api/openaiErrorClassification.ts — keep the two in sync. +export function isThrottle403(statusCode: number, body: string): boolean { + if (statusCode !== 403) return false + const lower = body.toLowerCase() + if ( + lower.includes('quota') || + lower.includes('exhausted') || + lower.includes('usage limit') || + lower.includes('usage_not_included') || + lower.includes('freeusagelimit') || + lower.includes('credit balance') || + lower.includes('billing limit') || + lower.includes('insufficient_quota') || + lower.includes('payment required') || + lower.includes('allotment') || + lower.includes('limit: 0') + ) + return false // billing stays terminal + if ( + /unauthorized|invalid.{0,10}(key|token)|api key|forbidden|access denied|revoked|expired|permission|not allowed|org_|insufficient_quota|auth/.test( + lower, + ) + ) + return false + const trimmed = body.trim() + if (trimmed === '') return true // the Zen volume-gating shape + return /rate.?limit|too many requests|try again|slow down|throttl|overloaded|capacity|busy/.test( + lower, + ) +} + export function parseRetryAfterHeaderMs(headers: Record, now: number): number | null { const retryAfterMs = headers['retry-after-ms'] if (retryAfterMs) { diff --git a/src/proxy/server.ts b/src/proxy/server.ts index 35f9378..e0167db 100644 --- a/src/proxy/server.ts +++ b/src/proxy/server.ts @@ -18,7 +18,7 @@ import { import { buildUpstreamHeaders, extractCacheHeaders } from './header-passthrough.js' import { applyCodeafCompat, type HarnessStamp } from './codeaf-compat.js' import { isChatCompletionsPath, toResponsesPath, toResponsesRequestBody, toChatCompletion, SseTranslator, type TranslatorStats } from './zen-responses.js' -import { isLocalNetworkOutage, isQuota429, parseRetryAfterHeaderMs, resolveCooldownMs } from './quota-detector.js' +import { isLocalNetworkOutage, isQuota429, isThrottle403, parseRetryAfterHeaderMs, resolveCooldownMs } from './quota-detector.js' import { parseUsageData } from './response-parser.js' import { estimateCost } from './rate-card.js' import { SessionAffinityStore } from './session-affinity.js' @@ -501,6 +501,50 @@ export class ProxyServer { } } + if (upstreamRes.status === 403) { + const throttleBody = await responseTextPromise + if (isThrottle403(upstreamRes.status, throttleBody)) { + const throttleCooldownMs = resolveCooldownMs(Object.fromEntries(upstreamRes.headers), throttleBody, Date.now(), 60_000) + // NOTE: no QuotaErrorSignal — markExhausted with a signal increments + // quotaErrorCount (key-manager.ts:177-186) and would pollute quota + // dashboards + notifier.keyExhausted. Throttle is not quota. + this.keyManager.markExhausted(key.id, throttleCooldownMs) + this.circuitBreaker.recordFailure(key.id) + this.keyManager.markError(key.id) + this.keyManager.recordRequest(key.id, { + statusCode: upstreamRes.status, + durationMs: duration, + model: prepared.model, + sessionId: upstreamSessionId ?? sessionKey ?? null, + successful: false, + }) + attemptedKeyIds.add(key.id) + this.logStream.emit( + this.logger, + 'warn', + `Key "${key.alias}" throttled (HTTP 403), cooldown ${throttleCooldownMs}ms, failing over`, + { + method: req.method, + harness, compatApplied, clientUa, + path: targetPath, + statusCode: upstreamRes.status, + keyAlias: key.alias, + keyId: key.id, + duration, + model: prepared.model, + strategy, + routeReason: reason, + selectedBySession, + sessionId: upstreamSessionId ?? sessionKey ?? null, + cooldownMs: throttleCooldownMs, + attempt: attempt + 1, + upstream: isZenRequest ? 'zen' : 'go', + }, + ) + if (attempt < maxAttempts - 1) continue + } + } // else: existing count-only path below, unchanged + if (upstreamRes.status >= 500) { const repeatedServerError = upstreamServerErrorSeen upstreamServerErrorSeen = true From f85cb2ef33a72467b16e68deda2696b77a50620d Mon Sep 17 00:00:00 2001 From: Reinis Date: Sat, 10 Oct 2026 13:07:41 +0300 Subject: [PATCH 2/2] fix(proxy): single-count last-attempt throttle 403 Branch recordRequest now runs on failover attempts only; on the last attempt the tail recordRequest counts the verbatim 403 once. Avoids requestCount/errorCount double-count and latency skew. Cooldown, breaker, markError and affinity protection still apply on every throttle hit. --- src/proxy/server.ts | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/proxy/server.ts b/src/proxy/server.ts index e0167db..bf4a93d 100644 --- a/src/proxy/server.ts +++ b/src/proxy/server.ts @@ -511,13 +511,17 @@ export class ProxyServer { this.keyManager.markExhausted(key.id, throttleCooldownMs) this.circuitBreaker.recordFailure(key.id) this.keyManager.markError(key.id) - this.keyManager.recordRequest(key.id, { - statusCode: upstreamRes.status, - durationMs: duration, - model: prepared.model, - sessionId: upstreamSessionId ?? sessionKey ?? null, - successful: false, - }) + if (attempt < maxAttempts - 1) { + // Counted here; on the last attempt the tail recordRequest + // below counts the verbatim 403 once (no double-count). + this.keyManager.recordRequest(key.id, { + statusCode: upstreamRes.status, + durationMs: duration, + model: prepared.model, + sessionId: upstreamSessionId ?? sessionKey ?? null, + successful: false, + }) + } attemptedKeyIds.add(key.id) this.logStream.emit( this.logger,