From fe1ad01579afd997b30fd56d49362a026ea035a1 Mon Sep 17 00:00:00 2001 From: Reinis Date: Sun, 11 Oct 2026 00:58:02 +0300 Subject: [PATCH] fix(native-retry): retry transient Zen throttle-403 on the native client path Unmarked 403s from the Zen lane travel the native Anthropic client, so the shim-only marker branch never fires for them. Add one additive shouldRetry branch (after the marker branch, before the CCR bypass): status 403, no marker, not Go-quota, not quota-exhausted, Zen-URL gated (x-opencode-request-url, OPENAI_BASE_URL, ANTHROPIC_BASE_URL), and the SDK-unwrapped body matches the reused isTransientThrottleMessage. All else falls through unchanged: isRetryableRoutedModelError, 401, Go-quota suites, analytics mapping untouched. Design + DoD: docs/openclaude/issues/2026-10-10-zen-403-audit.md (Second-patch implementation record). Tests: 9 new mirror tests in withRetry.test.ts (3 retry w/ countdown, 5 terminal attempts===1, 401 unchanged); focused 173 pass / 0 fail; typecheck byte-identical to pristine main; build green; related net 451 pass / 1 pre-existing fail (also fails on pristine main). --- src/services/api/openaiErrorClassification.ts | 2 +- src/services/api/withRetry.test.ts | 182 ++++++++++++++++++ src/services/api/withRetry.ts | 58 ++++++ 3 files changed, 241 insertions(+), 1 deletion(-) diff --git a/src/services/api/openaiErrorClassification.ts b/src/services/api/openaiErrorClassification.ts index 6d501c783d..b097deeb10 100644 --- a/src/services/api/openaiErrorClassification.ts +++ b/src/services/api/openaiErrorClassification.ts @@ -413,7 +413,7 @@ function isQuotaExhaustedMessage(body: string): boolean { ) } -function isTransientThrottleMessage(body: string): boolean { +export function isTransientThrottleMessage(body: string): boolean { // NOTE: mirrored by isThrottle403 in opencode-go-multi-auth // src/proxy/quota-detector.ts — keep the two in sync (comment cross-ref). const lower = body.toLowerCase() diff --git a/src/services/api/withRetry.test.ts b/src/services/api/withRetry.test.ts index 9caeffefab..578a48f601 100644 --- a/src/services/api/withRetry.test.ts +++ b/src/services/api/withRetry.test.ts @@ -36,6 +36,7 @@ const envKeys = [ 'OPENAI_MODEL', 'OPENAI_BASE_URL', 'OPENAI_API_BASE', + 'ANTHROPIC_BASE_URL', ] as const beforeEach(async () => { @@ -689,6 +690,187 @@ describe('OpenAI-compatible retry classification', () => { }) }) +describe('Native-path transient 403 retry (Zen volume gating)', () => { + // Mirror of the shim-path marker test above, for 403s that arrive via the + // native Anthropic client: no [openai_category] marker, SDK-prefixed + // message (`"403 " + JSON`), Zen route proven by env/header gate. + const ZEN_NATIVE_URL = 'https://opencode.ai/zen/v1/messages' + + function makeNativeError( + status: number, + message: string, + headers: Record = {}, + ): APIError { + return APIError.generate(status, undefined, message, new Headers(headers)) + } + + function findCountdown(yielded: unknown[]) { + return yielded.find( + (message) => + typeof message === 'object' && + message !== null && + (message as { type?: unknown }).type === 'system' && + (message as { subtype?: unknown }).subtype === 'api_error', + ) as { retryAttempt?: unknown; maxRetries?: unknown } | undefined + } + + async function runOnceThenSucceed(error: APIError) { + process.env.OPENCLAUDE_RETRY_DELAY_MS = '1' + const { withRetry } = await importFreshWithRetryModule('firstParty') + let attempts = 0 + const yielded: unknown[] = [] + const generator = withRetry( + async () => ({} as Anthropic), + async () => { + attempts++ + if (attempts === 1) throw error + return { ok: true } + }, + { + maxRetries: 2, + model: 'muse-spark-1.3-contributor-free', + thinkingConfig: { type: 'disabled' }, + }, + ) + let result: unknown + while (true) { + const next = await generator.next() + if (next.done) { + result = next.value + break + } + yielded.push(next.value) + } + return { result, attempts, yielded } + } + + async function runAlwaysFail(error: APIError) { + process.env.OPENCLAUDE_RETRY_DELAY_MS = '1' + const { CannotRetryError, withRetry } = + await importFreshWithRetryModule('firstParty') + let attempts = 0 + let caught: unknown + try { + await drainAsyncGenerator( + withRetry( + async () => ({} as Anthropic), + async () => { + attempts++ + throw error + }, + { + maxRetries: 2, + model: 'muse-spark-1.3-contributor-free', + thinkingConfig: { type: 'disabled' }, + }, + ), + ) + } catch (e) { + caught = e + } + expect(caught).toBeInstanceOf(CannotRetryError) + return { caught, attempts } + } + + test('retries an empty-body native 403 on the Zen lane with a countdown', async () => { + process.env.ANTHROPIC_BASE_URL = ZEN_NATIVE_URL + const { result, attempts, yielded } = await runOnceThenSucceed( + makeNativeError(403, '403 '), + ) + expect(result).toEqual({ ok: true }) + expect(attempts).toBe(2) + const countdown = findCountdown(yielded) + expect(countdown).toBeDefined() + expect(countdown?.retryAttempt).toBe(1) + expect(countdown?.maxRetries).toBe(2) + }) + + test('retries a whitespace-body native 403 via the OPENAI_BASE_URL gate', async () => { + process.env.OPENAI_BASE_URL = ZEN_NATIVE_URL + const { result, attempts, yielded } = await runOnceThenSucceed( + makeNativeError(403, '403 '), + ) + expect(result).toEqual({ ok: true }) + expect(attempts).toBe(2) + expect(findCountdown(yielded)).toBeDefined() + }) + + test('retries a throttle-phrased native 403 via the request-url header gate', async () => { + const { result, attempts, yielded } = await runOnceThenSucceed( + makeNativeError( + 403, + '403 {"type":"error","error":{"type":"throttled","message":"Please slow down and try again shortly."}}', + { 'x-opencode-request-url': ZEN_NATIVE_URL }, + ), + ) + expect(result).toEqual({ ok: true }) + expect(attempts).toBe(2) + expect(findCountdown(yielded)).toBeDefined() + }) + + test('stays terminal on a forbidden-worded native 403 (attempts===1)', async () => { + process.env.ANTHROPIC_BASE_URL = ZEN_NATIVE_URL + const { attempts } = await runAlwaysFail( + makeNativeError( + 403, + '403 {"type":"error","error":{"type":"forbidden","message":"Forbidden"}}', + ), + ) + expect(attempts).toBe(1) + }) + + test('stays terminal on a revoked-key native 403 (attempts===1)', async () => { + process.env.ANTHROPIC_BASE_URL = ZEN_NATIVE_URL + const { attempts } = await runAlwaysFail( + makeNativeError( + 403, + '403 {"type":"error","error":{"type":"authentication_error","message":"This API key has been revoked."}}', + ), + ) + expect(attempts).toBe(1) + }) + + test('stays terminal on an org-access native 403 (attempts===1)', async () => { + process.env.ANTHROPIC_BASE_URL = ZEN_NATIVE_URL + const { attempts } = await runAlwaysFail( + makeNativeError( + 403, + '403 {"type":"error","error":{"type":"permission_error","message":"Your organization does not have access to this model."}}', + ), + ) + expect(attempts).toBe(1) + }) + + test('stays terminal on a quota-worded native 403 (attempts===1)', async () => { + process.env.ANTHROPIC_BASE_URL = ZEN_NATIVE_URL + const { attempts } = await runAlwaysFail( + makeNativeError( + 403, + '403 {"type":"error","error":{"type":"rate_limit_error","message":"You exceeded your current quota, please check your plan and billing details."}}', + ), + ) + expect(attempts).toBe(1) + }) + + test('stays terminal for an empty-body native 403 off the Zen lane (attempts===1)', async () => { + process.env.ANTHROPIC_BASE_URL = 'https://api.anthropic.com' + const { attempts } = await runAlwaysFail(makeNativeError(403, '403 ')) + expect(attempts).toBe(1) + }) + + test('leaves native 401 retry behavior unchanged', async () => { + process.env.ANTHROPIC_BASE_URL = ZEN_NATIVE_URL + const { result, attempts } = await runOnceThenSucceed( + makeNativeError( + 401, + '401 {"type":"error","error":{"type":"authentication_error","message":"invalid x-api-key"}}', + ), + ) + expect(result).toEqual({ ok: true }) + expect(attempts).toBe(2) + }) +}) + // --- parseOpenAIDuration --- describe('parseOpenAIDuration', () => { test('parses seconds: "1s" → 1000', async () => { diff --git a/src/services/api/withRetry.ts b/src/services/api/withRetry.ts index 1206fed712..28df8239f3 100644 --- a/src/services/api/withRetry.ts +++ b/src/services/api/withRetry.ts @@ -55,6 +55,7 @@ import { extractOpenAICategoryMarker, isOpenAIRequestNonReplayable, isRetryableOpenAICompatibilityFailureCategory, + isTransientThrottleMessage, } from './openaiErrorClassification.js' const abortError = () => new APIUserAbortError() @@ -853,6 +854,47 @@ function handleGcpCredentialError(error: unknown): boolean { return false } +/** + * Whether the failing request was routed through the OpenCode Zen gateway. + * The native Anthropic client never sets `x-opencode-request-url` (shim-only + * plumbing), and the Zen lane keys off `ANTHROPIC_BASE_URL`, so all three + * signals are checked — any one carrying the Zen host is enough. + */ +function isZenRoutedRequest(error: APIError): boolean { + const requestUrl = error.headers?.get?.('x-opencode-request-url') ?? '' + return [requestUrl, process.env.OPENAI_BASE_URL ?? '', process.env.ANTHROPIC_BASE_URL ?? ''].some( + (candidate) => candidate.includes('opencode.ai/zen'), + ) +} + +/** + * Unwrap a native-client 403 body for the throttle matcher. The SDK never + * leaves `error.message` empty — it prepends the status (`"403 " + JSON`) — + * so strip that prefix, parse the envelope, and match the inner message. + * A missing or blank inner message is the empty Zen volume-gating shape. + */ +function extractNativeThrottleBody(message: string): string { + const stripped = message.replace(/^\d{3}\s+/, '') + let parsed: unknown + try { + parsed = JSON.parse(stripped) + } catch { + return stripped + } + if (parsed !== null && typeof parsed === 'object') { + const record = parsed as Record + const nested = record.error + if (typeof nested === 'string') return nested + if (nested !== null && typeof nested === 'object') { + const nestedMessage = (nested as Record).message + if (typeof nestedMessage === 'string') return nestedMessage + } + if (typeof record.message === 'string') return record.message + } + if (typeof parsed === 'string') return parsed + return '' +} + function shouldRetry(error: APIError, persistentRetryEnabled: boolean): boolean { // Never retry mock errors - they're from /mock-limits command for testing if (isMockRateLimitError(error)) { @@ -904,6 +946,22 @@ function shouldRetry(error: APIError, persistentRetryEnabled: boolean): boolean return true } + // Native-path transient throttle-403 (Zen volume gating). The Zen lane + // travels the native Anthropic client, so these 403s arrive unmarked and + // would otherwise fall through to terminal. Unwrap the SDK-prefixed body, + // require the Zen route, and reuse the shim's throttle matcher — anything + // else (other providers, auth-shaped bodies) falls through unchanged. + if ( + error.status === 403 && + !openAICategory && + !isOpenCodeGoQuotaError(error) && + !isQuotaExhausted(error) && + isZenRoutedRequest(error) && + isTransientThrottleMessage(extractNativeThrottleBody(error.message ?? '')) + ) { + return true + } + // CCR mode: auth is via infrastructure-provided JWTs, so a 401/403 is a // transient blip (auth service flap, network hiccup) rather than bad // credentials. Bypass x-should-retry:false — the server assumes we'd retry