diff --git a/src/proxy/quota-detector.ts b/src/proxy/quota-detector.ts index d0a251d..7767e64 100644 --- a/src/proxy/quota-detector.ts +++ b/src/proxy/quota-detector.ts @@ -37,6 +37,38 @@ export function isQuota429(statusCode: number, headers: Record, } } +// NOTE: mirrored by isTransientThrottleMessage in openclaude +// src/services/api/openaiErrorClassification.ts — keep the two in sync. +export function isThrottle403(statusCode: number, body: string): boolean { + if (statusCode !== 403) return false + const lower = body.toLowerCase() + if ( + lower.includes('quota') || + lower.includes('exhausted') || + lower.includes('usage limit') || + lower.includes('usage_not_included') || + lower.includes('freeusagelimit') || + lower.includes('credit balance') || + lower.includes('billing limit') || + lower.includes('insufficient_quota') || + lower.includes('payment required') || + lower.includes('allotment') || + lower.includes('limit: 0') + ) + return false // billing stays terminal + if ( + /unauthorized|invalid.{0,10}(key|token)|api key|forbidden|access denied|revoked|expired|permission|not allowed|org_|insufficient_quota|auth/.test( + lower, + ) + ) + return false + const trimmed = body.trim() + if (trimmed === '') return true // the Zen volume-gating shape + return /rate.?limit|too many requests|try again|slow down|throttl|overloaded|capacity|busy/.test( + lower, + ) +} + export function parseRetryAfterHeaderMs(headers: Record, now: number): number | null { const retryAfterMs = headers['retry-after-ms'] if (retryAfterMs) { diff --git a/src/proxy/server.ts b/src/proxy/server.ts index 35f9378..bf4a93d 100644 --- a/src/proxy/server.ts +++ b/src/proxy/server.ts @@ -18,7 +18,7 @@ import { import { buildUpstreamHeaders, extractCacheHeaders } from './header-passthrough.js' import { applyCodeafCompat, type HarnessStamp } from './codeaf-compat.js' import { isChatCompletionsPath, toResponsesPath, toResponsesRequestBody, toChatCompletion, SseTranslator, type TranslatorStats } from './zen-responses.js' -import { isLocalNetworkOutage, isQuota429, parseRetryAfterHeaderMs, resolveCooldownMs } from './quota-detector.js' +import { isLocalNetworkOutage, isQuota429, isThrottle403, parseRetryAfterHeaderMs, resolveCooldownMs } from './quota-detector.js' import { parseUsageData } from './response-parser.js' import { estimateCost } from './rate-card.js' import { SessionAffinityStore } from './session-affinity.js' @@ -501,6 +501,54 @@ export class ProxyServer { } } + if (upstreamRes.status === 403) { + const throttleBody = await responseTextPromise + if (isThrottle403(upstreamRes.status, throttleBody)) { + const throttleCooldownMs = resolveCooldownMs(Object.fromEntries(upstreamRes.headers), throttleBody, Date.now(), 60_000) + // NOTE: no QuotaErrorSignal — markExhausted with a signal increments + // quotaErrorCount (key-manager.ts:177-186) and would pollute quota + // dashboards + notifier.keyExhausted. Throttle is not quota. + this.keyManager.markExhausted(key.id, throttleCooldownMs) + this.circuitBreaker.recordFailure(key.id) + this.keyManager.markError(key.id) + if (attempt < maxAttempts - 1) { + // Counted here; on the last attempt the tail recordRequest + // below counts the verbatim 403 once (no double-count). + this.keyManager.recordRequest(key.id, { + statusCode: upstreamRes.status, + durationMs: duration, + model: prepared.model, + sessionId: upstreamSessionId ?? sessionKey ?? null, + successful: false, + }) + } + attemptedKeyIds.add(key.id) + this.logStream.emit( + this.logger, + 'warn', + `Key "${key.alias}" throttled (HTTP 403), cooldown ${throttleCooldownMs}ms, failing over`, + { + method: req.method, + harness, compatApplied, clientUa, + path: targetPath, + statusCode: upstreamRes.status, + keyAlias: key.alias, + keyId: key.id, + duration, + model: prepared.model, + strategy, + routeReason: reason, + selectedBySession, + sessionId: upstreamSessionId ?? sessionKey ?? null, + cooldownMs: throttleCooldownMs, + attempt: attempt + 1, + upstream: isZenRequest ? 'zen' : 'go', + }, + ) + if (attempt < maxAttempts - 1) continue + } + } // else: existing count-only path below, unchanged + if (upstreamRes.status >= 500) { const repeatedServerError = upstreamServerErrorSeen upstreamServerErrorSeen = true