fix(oracle-fusion): retry transient transport errors

Author: Bill Leoutsakos

apps/sim/lib/internal/oracle-fusion-financials/client.ts

@@ -15,6 +15,14 @@ const REQUEST_TIMEOUT_MS = 30_000
 const RESPONSE_MAX_BYTES = 5 * 1024 * 1024
 const MAX_RETRIES = 2
 const TRANSIENT_STATUSES = new Set([429, 503, 504])
+const TRANSIENT_TRANSPORT_CODES = new Set([
+  'ECONNREFUSED',
+  'ECONNRESET',
+  'EHOSTUNREACH',
+  'ENETUNREACH',
+  'EPIPE',
+  'ETIMEDOUT',
+])
 const LOSSLESS_DECIMAL_FIELDS = new Set([
   'InvoiceId',
   'InvoiceDistributionId',
@@ -95,6 +103,26 @@ function parseOracleFusionJson(body: string): unknown {
   })
 }
 
+function isTransientTransportError(error: unknown): boolean {
+  if (!(error instanceof Error)) return false
+  if (error.message === `Request timed out after ${REQUEST_TIMEOUT_MS}ms`) return true
+  return (
+    'code' in error && typeof error.code === 'string' && TRANSIENT_TRANSPORT_CODES.has(error.code)
+  )
+}
+
+async function waitForRetry(
+  attempt: number,
+  signal?: AbortSignal,
+  retryAfterMs: number | null = null
+) {
+  await interruptibleSleep(
+    backoffWithJitter(attempt + 1, retryAfterMs, { baseMs: 250, maxMs: 5_000 }),
+    signal
+  )
+  signal?.throwIfAborted()
+}
+
 export interface OracleFusionRequest {
   path: string
   query?: Record<string, string | number | boolean | undefined>
@@ -126,7 +154,7 @@ async function fetchAttempt(
   })
 }
 
-/** Executes one bounded, credential-bound Oracle GET with transient-status retries. */
+/** Executes one bounded, credential-bound Oracle GET with transient retries. */
 export async function requestOracleFusionJson(
   auth: Pick<OracleFusionAuthInput, 'accessToken' | 'instanceUrl'>,
   request: OracleFusionRequest,
@@ -154,27 +182,31 @@ export async function requestOracleFusionJson(
     let response: SecureFetchResponse
     try {
       response = await fetchAttempt(url, validation.resolvedIP, auth.accessToken, signal)
-    } catch {
+    } catch (error) {
       signal?.throwIfAborted()
+      if (isTransientTransportError(error) && attempt < MAX_RETRIES) {
+        await waitForRetry(attempt, signal)
+        continue
+      }
       throw new Error('Could not reach Oracle Fusion Financials')
     }
 
     if (TRANSIENT_STATUSES.has(response.status) && attempt < MAX_RETRIES) {
       const retryAfterMs = parseRetryAfter(response.headers.get('retry-after'))
       await consumeOrCancelBody(response)
-      await interruptibleSleep(
-        backoffWithJitter(attempt + 1, retryAfterMs, { baseMs: 250, maxMs: 5_000 }),
-        signal
-      )
-      signal?.throwIfAborted()
+      await waitForRetry(attempt, signal, retryAfterMs)
       continue
     }
 
     let body: string
     try {
       body = await response.text()
-    } catch {
+    } catch (error) {
       signal?.throwIfAborted()
+      if (isTransientTransportError(error) && attempt < MAX_RETRIES) {
+        await waitForRetry(attempt, signal)
+        continue
+      }
       throw new OracleFusionFinancialsProviderError(
         'Oracle Fusion Financials response could not be read',
         502

apps/sim/lib/internal/oracle-fusion-financials/oracle-fusion-financials.test.ts

@@ -1080,6 +1080,66 @@ describe('Oracle Fusion Financials provider', () => {
     expect(mockSleep).toHaveBeenCalledTimes(2)
   })
 
+  it('retries classified transient transport failures within the existing bound', async () => {
+    mockSecureFetch
+      .mockRejectedValueOnce(Object.assign(new Error('socket reset'), { code: 'ECONNRESET' }))
+      .mockRejectedValueOnce(new Error('Request timed out after 30000ms'))
+      .mockResolvedValueOnce(response(200, page([])))
+
+    await requestOracleFusionJson(AUTH, { path: `${RESOURCE_PATH}/invoices` })
+
+    expect(mockSecureFetch).toHaveBeenCalledTimes(3)
+    expect(mockBackoff).toHaveBeenNthCalledWith(1, 1, null, {
+      baseMs: 250,
+      maxMs: 5_000,
+    })
+    expect(mockBackoff).toHaveBeenNthCalledWith(2, 2, null, {
+      baseMs: 250,
+      maxMs: 5_000,
+    })
+    expect(mockSleep).toHaveBeenCalledTimes(2)
+  })
+
+  it('stops after two transient transport retries and keeps the failure sanitized', async () => {
+    const reset = () => Object.assign(new Error('provider-host-canary'), { code: 'ECONNRESET' })
+    mockSecureFetch
+      .mockRejectedValueOnce(reset())
+      .mockRejectedValueOnce(reset())
+      .mockRejectedValueOnce(reset())
+
+    await expect(
+      requestOracleFusionJson(AUTH, { path: `${RESOURCE_PATH}/invoices` })
+    ).rejects.toThrow('Could not reach Oracle Fusion Financials')
+    expect(mockSecureFetch).toHaveBeenCalledTimes(3)
+    expect(mockSleep).toHaveBeenCalledTimes(2)
+  })
+
+  it('retries a classified transport failure while reading the response body', async () => {
+    mockSecureFetch
+      .mockResolvedValueOnce({
+        ...response(200, {}),
+        text: async () => {
+          throw Object.assign(new Error('socket reset'), { code: 'ECONNRESET' })
+        },
+      })
+      .mockResolvedValueOnce(response(200, page([])))
+
+    await requestOracleFusionJson(AUTH, { path: `${RESOURCE_PATH}/invoices` })
+
+    expect(mockSecureFetch).toHaveBeenCalledTimes(2)
+    expect(mockSleep).toHaveBeenCalledTimes(1)
+  })
+
+  it('does not retry unclassified transport failures', async () => {
+    mockSecureFetch.mockRejectedValueOnce(new TypeError('invalid pinned request'))
+
+    await expect(
+      requestOracleFusionJson(AUTH, { path: `${RESOURCE_PATH}/invoices` })
+    ).rejects.toThrow('Could not reach Oracle Fusion Financials')
+    expect(mockSecureFetch).toHaveBeenCalledTimes(1)
+    expect(mockSleep).not.toHaveBeenCalled()
+  })
+
   it('stops after two retries and surfaces a sanitized Oracle error', async () => {
     const accessToken = 'short/lived+access~token='
     const encodedAccessToken = encodeURIComponent(accessToken)
@@ -1127,6 +1187,22 @@ describe('Oracle Fusion Financials provider', () => {
       requestOracleFusionJson(AUTH, { path: `${RESOURCE_PATH}/invoices` }, duringRetry.signal)
     ).rejects.toMatchObject({ name: 'AbortError' })
     expect(mockSecureFetch).toHaveBeenCalledTimes(1)
+
+    const duringTransportRetry = new AbortController()
+    mockSecureFetch.mockRejectedValueOnce(
+      Object.assign(new Error('socket reset'), { code: 'ECONNRESET' })
+    )
+    mockSleep.mockImplementationOnce(async () => {
+      duringTransportRetry.abort(new DOMException('cancelled', 'AbortError'))
+    })
+    await expect(
+      requestOracleFusionJson(
+        AUTH,
+        { path: `${RESOURCE_PATH}/invoices` },
+        duringTransportRetry.signal
+      )
+    ).rejects.toMatchObject({ name: 'AbortError' })
+    expect(mockSecureFetch).toHaveBeenCalledTimes(2)
   })
 
   it('maps invalid caller input to 400 and malformed Oracle responses to a sanitized 502', async () => {