fix(connectors): stop hydration after rate limits (#7255)

* fix(connectors): stop hydration after rate limits

* fix(connectors): normalize provider throttles

* fix(connectors): inspect all drive error reasons

Author: waleedlatif1

apps/sim/connectors/github/github.test.ts

@@ -0,0 +1,120 @@
+/**
+ * @vitest-environment node
+ */
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { githubConnector } from '@/connectors/github/github'
+
+describe('githubConnector.getDocument', () => {
+  afterEach(() => {
+    vi.unstubAllGlobals()
+  })
+
+  it('uses the object media type and hydrates large file content through the blob API', async () => {
+    const fetchMock = vi
+      .fn()
+      .mockResolvedValueOnce(
+        new Response(
+          JSON.stringify({
+            sha: 'blob-sha',
+            size: 2 * 1024 * 1024,
+            content: '',
+            encoding: 'none',
+          }),
+          { status: 200, headers: { 'last-modified': 'Fri, 28 Aug 2026 12:00:00 GMT' } }
+        )
+      )
+      .mockResolvedValueOnce(
+        new Response('large text file', { status: 200, headers: { 'content-length': '15' } })
+      )
+    vi.stubGlobal('fetch', fetchMock)
+
+    const document = await githubConnector.getDocument(
+      'token',
+      { repository: 'owner/repo', branch: 'main' },
+      'docs/large.md'
+    )
+
+    expect(fetchMock).toHaveBeenCalledTimes(2)
+    expect(fetchMock.mock.calls[0][1]).toMatchObject({
+      headers: expect.objectContaining({ Accept: 'application/vnd.github.object+json' }),
+    })
+    expect(fetchMock.mock.calls[1][1]).toMatchObject({
+      headers: expect.objectContaining({ Accept: 'application/vnd.github.raw+json' }),
+    })
+    expect(document).toMatchObject({
+      externalId: 'docs/large.md',
+      content: 'large text file',
+      contentDeferred: false,
+      contentHash: 'git-sha:blob-sha',
+    })
+  })
+
+  it('returns null only when a listed path is no longer present', async () => {
+    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response(null, { status: 404 })))
+
+    await expect(
+      githubConnector.getDocument('token', { repository: 'owner/repo' }, 'deleted.md')
+    ).resolves.toBeNull()
+  })
+
+  it('records a blob that exceeds the byte cap as a visible skipped document', async () => {
+    const fetchMock = vi
+      .fn()
+      .mockResolvedValueOnce(
+        new Response(
+          JSON.stringify({
+            sha: 'blob-sha',
+            size: 2 * 1024 * 1024,
+            content: '',
+            encoding: 'none',
+          }),
+          { status: 200 }
+        )
+      )
+      .mockResolvedValueOnce(
+        new Response('oversized', {
+          status: 200,
+          headers: { 'content-length': String(100 * 1024 * 1024 + 1) },
+        })
+      )
+    vi.stubGlobal('fetch', fetchMock)
+
+    await expect(
+      githubConnector.getDocument('token', { repository: 'owner/repo' }, 'oversized.md')
+    ).resolves.toMatchObject({
+      externalId: 'oversized.md',
+      content: '',
+      skippedReason: 'File exceeds the 100MB size limit and was not indexed',
+    })
+  })
+
+  it('rejects a bodyless blob response instead of misreporting it as oversized', async () => {
+    const fetchMock = vi
+      .fn()
+      .mockResolvedValueOnce(
+        new Response(
+          JSON.stringify({
+            sha: 'blob-sha',
+            size: 2 * 1024 * 1024,
+            content: '',
+            encoding: 'none',
+          }),
+          { status: 200 }
+        )
+      )
+      .mockResolvedValueOnce(new Response(null, { status: 200 }))
+    vi.stubGlobal('fetch', fetchMock)
+
+    await expect(
+      githubConnector.getDocument('token', { repository: 'owner/repo' }, 'missing-body.md')
+    ).rejects.toThrow('GitHub git blob blob-sha returned no body')
+  })
+
+  it('surfaces a non-rate-limit 403 as a document failure', async () => {
+    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response(null, { status: 403 })))
+
+    await expect(
+      githubConnector.getDocument('token', { repository: 'owner/repo' }, 'private.md')
+    ).rejects.toThrow('Failed to fetch file private.md: 403')
+  })
+})

apps/sim/connectors/github/github.ts

@@ -5,8 +5,10 @@ import { githubConnectorMeta } from '@/connectors/github/meta'
 import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types'
 import {
   CONNECTOR_MAX_FILE_BYTES,
+  ConnectorFileTooLargeError,
   markSkipped,
   parseTagDate,
+  readBodyWithLimit,
   sizeLimitSkipReason,
   stubOrSkipBySize,
   takeIndexableWithinCap,
@@ -156,7 +158,7 @@ async function fetchBlobContent(
   const response = await fetchWithRetry(url, {
     method: 'GET',
     headers: {
-      Accept: 'application/vnd.github+json',
+      Accept: 'application/vnd.github.raw+json',
       Authorization: `Bearer ${accessToken}`,
       'X-GitHub-Api-Version': '2022-11-28',
     },
@@ -166,25 +168,20 @@ async function fetchBlobContent(
     throw new Error(`Failed to fetch git blob ${sha}: ${response.status}`)
   }
 
-  const data = await response.json()
-  const content = (data.content as string) || ''
-  const encoding = data.encoding as string | undefined
+  if (!response.body) {
+    const contentLength = Number.parseInt(response.headers.get('content-length') ?? '', 10)
+    if (Number.isFinite(contentLength) && contentLength > MAX_FILE_SIZE) {
+      throw new ConnectorFileTooLargeError(MAX_FILE_SIZE)
+    }
+    throw new Error(`GitHub git blob ${sha} returned no body`)
+  }
 
-  if (encoding === 'base64') {
-    const buf = Buffer.from(content, 'base64')
-    if (isBinaryBuffer(buf)) return null
-    return buf.toString('utf8')
+  const buffer = await readBodyWithLimit(response, MAX_FILE_SIZE)
+  if (!buffer) {
+    throw new ConnectorFileTooLargeError(MAX_FILE_SIZE)
   }
-  /**
-   * `GET /repos/{owner}/{repo}/git/blobs/{sha}` documents a single response
-   * encoding: "The `content` in the response will always be Base64 encoded."
-   * The "Currently, `utf-8` and `base64` are supported" sentence belongs to the
-   * `encoding` REQUEST parameter of `POST .../git/blobs` (Create a blob) and does
-   * not describe this response, so no `utf-8` branch is warranted here. Any other
-   * encoding would silently persist empty content, so it throws and surfaces as a
-   * failed document instead.
-   */
-  throw new Error(`Unexpected git blob encoding for ${sha}: ${encoding ?? 'undefined'}`)
+  if (isBinaryBuffer(buffer)) return null
+  return buffer.toString('utf8')
 }
 
 /**
@@ -325,33 +322,14 @@ export const githubConnector: ConnectorConfig = {
       const response = await fetchWithRetry(url, {
         method: 'GET',
         headers: {
-          Accept: 'application/vnd.github+json',
+          Accept: 'application/vnd.github.object+json',
           Authorization: `Bearer ${accessToken}`,
           'X-GitHub-Api-Version': '2022-11-28',
         },
       })
 
       if (!response.ok) {
         if (response.status === 404) return null
-        /**
-         * A rate-limit 403 never reaches here: `fetchWithRetry` treats a 403 carrying
-         * `retry-after` or `x-ratelimit-remaining: 0` as retryable and throws once the
-         * retries are spent, so it lands in the catch below as a failure.
-         *
-         * A 403 that survives is usually an authorization denial, but NOT always: this
-         * request sends `application/vnd.github+json`, and the Contents API documents
-         * that files between 1-100 MB support "only the `raw` or `object` custom media
-         * types". A >1 MB text file therefore also lands here and is dropped, which on
-         * an `add` is silent (a fulfilled `null` records no failure). Reconciliation is
-         * unaffected — the file is already in `seenExternalIds` from the listing.
-         */
-        if (response.status === 403) {
-          logger.info('Skipping GitHub file rejected by Contents API', {
-            path,
-            status: response.status,
-          })
-          return null
-        }
         throw new Error(`Failed to fetch file ${path}: ${response.status}`)
       }
 
@@ -392,15 +370,18 @@ export const githubConnector: ConnectorConfig = {
          * "only the `raw` or `object` custom media types are supported", and it is
          * specifically "when using the `object` media type" that "the `content` field
          * will be an empty string and the `encoding` field will be `none`".
-         *
-         * This request sends `application/vnd.github+json`, so that precondition does
-         * not hold and this branch is currently unreachable — such files 403 above
-         * instead. Reaching it would require requesting
-         * `application/vnd.github.object+json`. The fallback itself is correct: the Git
-         * Blobs API returns the same blob as JSON and is documented to support blobs up
-         * to 100 MB.
+         * The Git Blobs fallback streams that same blob through GitHub's documented raw
+         * media type and supports blobs up to 100 MB.
          */
-        const blobContent = await fetchBlobContent(accessToken, owner, repo, data.sha as string)
+        let blobContent: string | null
+        try {
+          blobContent = await fetchBlobContent(accessToken, owner, repo, data.sha as string)
+        } catch (error) {
+          if (error instanceof ConnectorFileTooLargeError) {
+            return markSkipped(stub, sizeLimitSkipReason(MAX_FILE_SIZE))
+          }
+          throw error
+        }
         if (blobContent === null) {
           logger.info('Skipping binary GitHub file', { path, size })
           return markSkipped(stub, BINARY_SKIP_REASON)

apps/sim/connectors/google-drive/google-drive-errors.ts

@@ -12,13 +12,12 @@ const PERMISSION_REASONS = new Set([
 const POLICY_REASONS = new Set(['domainPolicy', 'download_restricted_for_revision'])
 const UNSUPPORTED_EXPORT_REASONS = new Set(['fileNotDownloadable', 'fileNotExportable'])
 const QUOTA_REASONS = new Set(['dailyLimitExceeded', 'quotaExceeded'])
-const TRANSIENT_REASONS = new Set([
-  'backendError',
-  'internalError',
+const RATE_LIMIT_REASONS = new Set([
   'rateLimitExceeded',
   'sharingRateLimitExceeded',
   'userRateLimitExceeded',
 ])
+const TRANSIENT_REASONS = new Set(['backendError', 'internalError', ...RATE_LIMIT_REASONS])
 
 export type GoogleDriveErrorKind =
   | 'authorization'
@@ -99,15 +98,22 @@ function classifyGoogleDriveError(
 
 export class GoogleDriveApiError extends Error {
   retryAfterMs?: number
+  readonly reasons: readonly string[]
+  readonly kind: GoogleDriveErrorKind
+  readonly rateLimited: boolean
 
   constructor(
     readonly status: number,
-    readonly reasons: readonly string[],
-    readonly kind: GoogleDriveErrorKind
+    normalizedReasons: readonly string[]
   ) {
-    const reasonSuffix = reasons.length > 0 ? ` (${reasons.join(', ')})` : ''
+    const diagnosticReasons = normalizedReasons.slice(0, GOOGLE_ERROR_REASON_MAX_COUNT)
+    const reasonSuffix = diagnosticReasons.length > 0 ? ` (${diagnosticReasons.join(', ')})` : ''
     super(`Google Drive API request failed with HTTP ${status}${reasonSuffix}`)
     this.name = 'GoogleDriveApiError'
+    this.reasons = diagnosticReasons
+    this.kind = classifyGoogleDriveError(status, normalizedReasons)
+    this.rateLimited =
+      status === 429 || normalizedReasons.some((reason) => RATE_LIMIT_REASONS.has(reason))
   }
 }
 
@@ -130,13 +136,8 @@ export async function readGoogleDriveApiError(response: Response): Promise<Googl
 
   const entries = parsedBody?.error?.errors ?? []
   const rawReasons = [...new Set(entries.flatMap((entry) => (entry.reason ? [entry.reason] : [])))]
-  const reasons = [...new Set(rawReasons.flatMap((reason) => normalizeReason(reason) ?? []))].slice(
-    0,
-    GOOGLE_ERROR_REASON_MAX_COUNT
-  )
-  return new GoogleDriveApiError(
-    response.status,
-    reasons,
-    classifyGoogleDriveError(response.status, rawReasons)
-  )
+  const normalizedReasons = [
+    ...new Set(rawReasons.flatMap((reason) => normalizeReason(reason) ?? [])),
+  ]
+  return new GoogleDriveApiError(response.status, normalizedReasons)
 }

apps/sim/connectors/google-drive/google-drive.test.ts

@@ -94,6 +94,39 @@ describe('Google Drive API error parsing', () => {
     expect(error.message).not.toContain('upstream unavailable')
   })
 
+  it('normalizes only structured rate-limit reasons into the shared throttle signal', async () => {
+    const rateLimit = await readGoogleDriveApiError(
+      driveErrorResponse('userRateLimitExceeded', 'Provider message')
+    )
+    const backendFailure = await readGoogleDriveApiError(
+      driveErrorResponse('backendError', 'Provider message', 503)
+    )
+
+    expect(rateLimit.rateLimited).toBe(true)
+    expect(backendFailure.rateLimited).toBe(false)
+  })
+
+  it('detects a structured rate limit beyond the bounded diagnostic reasons', async () => {
+    const reasons = [
+      ...Array.from({ length: 16 }, (_, index) => `providerReason${index}`),
+      'userRateLimitExceeded',
+    ]
+    const error = await readGoogleDriveApiError(
+      jsonResponse(
+        {
+          error: {
+            errors: reasons.map((reason) => ({ reason })),
+          },
+        },
+        403
+      )
+    )
+
+    expect(error.reasons).toEqual(reasons.slice(0, 16))
+    expect(error.kind).toBe('transient')
+    expect(error.rateLimited).toBe(true)
+  })
+
   it('omits provider messages from diagnostics', async () => {
     const message = `Authorization: Bearer private-token\ncontext ${'x'.repeat(700)}`
     const error = await readGoogleDriveApiError(