Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/services/api/openaiErrorClassification.ts
Original file line number Diff line number Diff line change
Expand Up @@ -413,7 +413,7 @@ function isQuotaExhaustedMessage(body: string): boolean {
)
}

function isTransientThrottleMessage(body: string): boolean {
export function isTransientThrottleMessage(body: string): boolean {
// NOTE: mirrored by isThrottle403 in opencode-go-multi-auth
// src/proxy/quota-detector.ts — keep the two in sync (comment cross-ref).
const lower = body.toLowerCase()
Expand Down
182 changes: 182 additions & 0 deletions src/services/api/withRetry.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ const envKeys = [
'OPENAI_MODEL',
'OPENAI_BASE_URL',
'OPENAI_API_BASE',
'ANTHROPIC_BASE_URL',
] as const

beforeEach(async () => {
Expand Down Expand Up @@ -689,6 +690,187 @@ describe('OpenAI-compatible retry classification', () => {
})
})

describe('Native-path transient 403 retry (Zen volume gating)', () => {
// Mirror of the shim-path marker test above, for 403s that arrive via the
// native Anthropic client: no [openai_category] marker, SDK-prefixed
// message (`"403 " + JSON`), Zen route proven by env/header gate.
const ZEN_NATIVE_URL = 'https://opencode.ai/zen/v1/messages'

function makeNativeError(
status: number,
message: string,
headers: Record<string, string> = {},
): APIError {
return APIError.generate(status, undefined, message, new Headers(headers))
}

function findCountdown(yielded: unknown[]) {
return yielded.find(
(message) =>
typeof message === 'object' &&
message !== null &&
(message as { type?: unknown }).type === 'system' &&
(message as { subtype?: unknown }).subtype === 'api_error',
) as { retryAttempt?: unknown; maxRetries?: unknown } | undefined
}

async function runOnceThenSucceed(error: APIError) {
process.env.OPENCLAUDE_RETRY_DELAY_MS = '1'
const { withRetry } = await importFreshWithRetryModule('firstParty')
let attempts = 0
const yielded: unknown[] = []
const generator = withRetry(
async () => ({} as Anthropic),
async () => {
attempts++
if (attempts === 1) throw error
return { ok: true }
},
{
maxRetries: 2,
model: 'muse-spark-1.3-contributor-free',
thinkingConfig: { type: 'disabled' },
},
)
let result: unknown
while (true) {
const next = await generator.next()
if (next.done) {
result = next.value
break
}
yielded.push(next.value)
}
return { result, attempts, yielded }
}

async function runAlwaysFail(error: APIError) {
process.env.OPENCLAUDE_RETRY_DELAY_MS = '1'
const { CannotRetryError, withRetry } =
await importFreshWithRetryModule('firstParty')
let attempts = 0
let caught: unknown
try {
await drainAsyncGenerator(
withRetry(
async () => ({} as Anthropic),
async () => {
attempts++
throw error
},
{
maxRetries: 2,
model: 'muse-spark-1.3-contributor-free',
thinkingConfig: { type: 'disabled' },
},
),
)
} catch (e) {
caught = e
}
expect(caught).toBeInstanceOf(CannotRetryError)
return { caught, attempts }
}

test('retries an empty-body native 403 on the Zen lane with a countdown', async () => {
process.env.ANTHROPIC_BASE_URL = ZEN_NATIVE_URL
const { result, attempts, yielded } = await runOnceThenSucceed(
makeNativeError(403, '403 '),
)
expect(result).toEqual({ ok: true })
expect(attempts).toBe(2)
const countdown = findCountdown(yielded)
expect(countdown).toBeDefined()
expect(countdown?.retryAttempt).toBe(1)
expect(countdown?.maxRetries).toBe(2)
})

test('retries a whitespace-body native 403 via the OPENAI_BASE_URL gate', async () => {
process.env.OPENAI_BASE_URL = ZEN_NATIVE_URL
const { result, attempts, yielded } = await runOnceThenSucceed(
makeNativeError(403, '403 '),
)
expect(result).toEqual({ ok: true })
expect(attempts).toBe(2)
expect(findCountdown(yielded)).toBeDefined()
})

test('retries a throttle-phrased native 403 via the request-url header gate', async () => {
const { result, attempts, yielded } = await runOnceThenSucceed(
makeNativeError(
403,
'403 {"type":"error","error":{"type":"throttled","message":"Please slow down and try again shortly."}}',
{ 'x-opencode-request-url': ZEN_NATIVE_URL },
),
)
expect(result).toEqual({ ok: true })
expect(attempts).toBe(2)
expect(findCountdown(yielded)).toBeDefined()
})

test('stays terminal on a forbidden-worded native 403 (attempts===1)', async () => {
process.env.ANTHROPIC_BASE_URL = ZEN_NATIVE_URL
const { attempts } = await runAlwaysFail(
makeNativeError(
403,
'403 {"type":"error","error":{"type":"forbidden","message":"Forbidden"}}',
),
)
expect(attempts).toBe(1)
})

test('stays terminal on a revoked-key native 403 (attempts===1)', async () => {
process.env.ANTHROPIC_BASE_URL = ZEN_NATIVE_URL
const { attempts } = await runAlwaysFail(
makeNativeError(
403,
'403 {"type":"error","error":{"type":"authentication_error","message":"This API key has been revoked."}}',
),
)
expect(attempts).toBe(1)
})

test('stays terminal on an org-access native 403 (attempts===1)', async () => {
process.env.ANTHROPIC_BASE_URL = ZEN_NATIVE_URL
const { attempts } = await runAlwaysFail(
makeNativeError(
403,
'403 {"type":"error","error":{"type":"permission_error","message":"Your organization does not have access to this model."}}',
),
)
expect(attempts).toBe(1)
})

test('stays terminal on a quota-worded native 403 (attempts===1)', async () => {
process.env.ANTHROPIC_BASE_URL = ZEN_NATIVE_URL
const { attempts } = await runAlwaysFail(
makeNativeError(
403,
'403 {"type":"error","error":{"type":"rate_limit_error","message":"You exceeded your current quota, please check your plan and billing details."}}',
),
)
expect(attempts).toBe(1)
})

test('stays terminal for an empty-body native 403 off the Zen lane (attempts===1)', async () => {
process.env.ANTHROPIC_BASE_URL = 'https://api.anthropic.com'
const { attempts } = await runAlwaysFail(makeNativeError(403, '403 '))
expect(attempts).toBe(1)
})

test('leaves native 401 retry behavior unchanged', async () => {
process.env.ANTHROPIC_BASE_URL = ZEN_NATIVE_URL
const { result, attempts } = await runOnceThenSucceed(
makeNativeError(
401,
'401 {"type":"error","error":{"type":"authentication_error","message":"invalid x-api-key"}}',
),
)
expect(result).toEqual({ ok: true })
expect(attempts).toBe(2)
})
})

// --- parseOpenAIDuration ---
describe('parseOpenAIDuration', () => {
test('parses seconds: "1s" → 1000', async () => {
Expand Down
58 changes: 58 additions & 0 deletions src/services/api/withRetry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ import {
extractOpenAICategoryMarker,
isOpenAIRequestNonReplayable,
isRetryableOpenAICompatibilityFailureCategory,
isTransientThrottleMessage,
} from './openaiErrorClassification.js'

const abortError = () => new APIUserAbortError()
Expand Down Expand Up @@ -853,6 +854,47 @@ function handleGcpCredentialError(error: unknown): boolean {
return false
}

/**
* Whether the failing request was routed through the OpenCode Zen gateway.
* The native Anthropic client never sets `x-opencode-request-url` (shim-only
* plumbing), and the Zen lane keys off `ANTHROPIC_BASE_URL`, so all three
* signals are checked — any one carrying the Zen host is enough.
*/
function isZenRoutedRequest(error: APIError): boolean {
const requestUrl = error.headers?.get?.('x-opencode-request-url') ?? ''
return [requestUrl, process.env.OPENAI_BASE_URL ?? '', process.env.ANTHROPIC_BASE_URL ?? ''].some(
(candidate) => candidate.includes('opencode.ai/zen'),
)
}

/**
* Unwrap a native-client 403 body for the throttle matcher. The SDK never
* leaves `error.message` empty — it prepends the status (`"403 " + JSON`) —
* so strip that prefix, parse the envelope, and match the inner message.
* A missing or blank inner message is the empty Zen volume-gating shape.
*/
function extractNativeThrottleBody(message: string): string {
const stripped = message.replace(/^\d{3}\s+/, '')
let parsed: unknown
try {
parsed = JSON.parse(stripped)
} catch {
return stripped
}
if (parsed !== null && typeof parsed === 'object') {
const record = parsed as Record<string, unknown>
const nested = record.error
if (typeof nested === 'string') return nested
if (nested !== null && typeof nested === 'object') {
const nestedMessage = (nested as Record<string, unknown>).message
if (typeof nestedMessage === 'string') return nestedMessage
}
if (typeof record.message === 'string') return record.message
}
if (typeof parsed === 'string') return parsed
return ''
}

function shouldRetry(error: APIError, persistentRetryEnabled: boolean): boolean {
// Never retry mock errors - they're from /mock-limits command for testing
if (isMockRateLimitError(error)) {
Expand Down Expand Up @@ -904,6 +946,22 @@ function shouldRetry(error: APIError, persistentRetryEnabled: boolean): boolean
return true
}

// Native-path transient throttle-403 (Zen volume gating). The Zen lane
// travels the native Anthropic client, so these 403s arrive unmarked and
// would otherwise fall through to terminal. Unwrap the SDK-prefixed body,
// require the Zen route, and reuse the shim's throttle matcher — anything
// else (other providers, auth-shaped bodies) falls through unchanged.
if (
error.status === 403 &&
!openAICategory &&
!isOpenCodeGoQuotaError(error) &&
!isQuotaExhausted(error) &&
isZenRoutedRequest(error) &&
isTransientThrottleMessage(extractNativeThrottleBody(error.message ?? ''))
) {
return true
}

// CCR mode: auth is via infrastructure-provided JWTs, so a 401/403 is a
// transient blip (auth service flap, network hiccup) rather than bad
// credentials. Bypass x-should-retry:false — the server assumes we'd retry
Expand Down
Loading