fix(knowledge): harden OCR and SharePoint ingestion limits (#7135)

* fix(connectors): fail closed on incomplete SharePoint traversal

* fix(knowledge): enforce OCR request limits by bytes and pages

* fix(knowledge): retain OCR policy literals

Author: waleedlatif1

apps/sim/app/api/tools/mistral/parse/route.ts

@@ -9,13 +9,17 @@ import {
   validateUrlWithDNS,
 } from '@/lib/core/security/input-validation.server'
 import { generateRequestId } from '@/lib/core/utils/request'
+import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 import { validateOpaqueModelInputProvenance } from '@/lib/execution/model-input-provenance'
+import { decodeDataUriWithinLimit } from '@/lib/file-parsers/data-uri'
+import { isFileParserError } from '@/lib/file-parsers/errors'
+import { MISTRAL_OCR_REQUEST_POLICY } from '@/lib/knowledge/documents/ocr-request-policy'
+import { readBoundedHttpErrorBody } from '@/lib/knowledge/documents/utils'
 import {
   isModelSafeWorkspaceFileKey,
   MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE,
 } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance'
-import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types'
 import {
   extractStorageKey,
   isInternalFileUrl,
@@ -160,14 +164,43 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
           requestId,
           logger,
           {
-            maxBytes: MAX_BUFFERED_TRANSFER_BYTES,
+            maxBytes: MISTRAL_OCR_REQUEST_POLICY.maxBytes,
           }
         )
         base64 = buffer.toString('base64')
         if (contentType && contentType !== 'application/octet-stream') {
           mimeType = contentType
         }
       }
+
+      let inlineBytes: number
+      try {
+        inlineBytes = base64.startsWith('data:')
+          ? decodeDataUriWithinLimit(base64, MISTRAL_OCR_REQUEST_POLICY.maxBytes).buffer.length
+          : Buffer.byteLength(base64, 'base64')
+      } catch (error) {
+        const status = isFileParserError(error) && error.code === 'complexity_limit' ? 413 : 400
+        return NextResponse.json(
+          {
+            success: false,
+            error:
+              status === 413
+                ? `File exceeds Mistral OCR's ${MISTRAL_OCR_REQUEST_POLICY.maxBytes.toLocaleString()}-byte request limit`
+                : getErrorMessage(error, 'Invalid inline file data'),
+          },
+          { status }
+        )
+      }
+      if (inlineBytes > MISTRAL_OCR_REQUEST_POLICY.maxBytes) {
+        return NextResponse.json(
+          {
+            success: false,
+            error: `File exceeds Mistral OCR's ${MISTRAL_OCR_REQUEST_POLICY.maxBytes.toLocaleString()}-byte request limit`,
+          },
+          { status: 413 }
+        )
+      }
+
       const base64Payload = base64.startsWith('data:')
         ? base64
         : `data:${mimeType};base64,${base64}`
@@ -295,8 +328,11 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
     )
 
     if (!mistralResponse.ok) {
-      const errorText = await mistralResponse.text()
-      logger.error(`[${requestId}] Mistral API error:`, errorText)
+      const errorText = await readBoundedHttpErrorBody(mistralResponse)
+      logger.error(`[${requestId}] Mistral API error`, {
+        status: mistralResponse.status,
+        diagnostic: errorText,
+      })
       return NextResponse.json(
         {
           success: false,
@@ -318,6 +354,16 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
     const notReady = docNotReadyResponse(error)
     if (notReady) return notReady
 
+    if (isPayloadSizeLimitError(error)) {
+      return NextResponse.json(
+        {
+          success: false,
+          error: `File exceeds Mistral OCR's ${MISTRAL_OCR_REQUEST_POLICY.maxBytes.toLocaleString()}-byte request limit`,
+        },
+        { status: 413 }
+      )
+    }
+
     logger.error(`[${requestId}] Error in Mistral parse:`, error)
 
     return NextResponse.json(

apps/sim/connectors/sharepoint/sharepoint.test.ts

@@ -105,6 +105,29 @@ function rootChildren(driveId: string, items: unknown[]) {
   }
 }
 
+/** Builds a Graph pagination chain with an optional continuation beyond the final allowed page. */
+function paginatedRoutes(
+  initialUrl: string,
+  routePrefix: string,
+  pageCount: number,
+  continueAfterLast: boolean
+): Record<string, GraphRoute> {
+  const routes: Record<string, GraphRoute> = {}
+
+  for (let page = 0; page < pageCount; page++) {
+    const url = page === 0 ? initialUrl : `${GRAPH}/${routePrefix}/${page}`
+    const hasNextPage = page < pageCount - 1 || continueAfterLast
+    routes[url] = {
+      body: {
+        value: [],
+        ...(hasNextPage ? { '@odata.nextLink': `${GRAPH}/${routePrefix}/${page + 1}` } : {}),
+      },
+    }
+  }
+
+  return routes
+}
+
 function resolve(folderPath?: string) {
   return resolveFolderTarget('token', SITE_ID, SITE_URL, 'Contoso', folderPath)
 }
@@ -322,6 +345,51 @@ describe('resolveFolderTarget', () => {
     )
   })
 
+  it('surfaces a terminal document-library listing failure', async () => {
+    mockGraph({
+      ...defaultDriveRoute,
+      [`${GRAPH}/sites/${SITE_ID}/drives?$select=id,name,webUrl`]: {
+        status: 503,
+        body: { error: { message: 'Service unavailable' } },
+      },
+    })
+
+    await expect(resolve('Missing')).rejects.toThrow(
+      /Failed to list SharePoint document libraries: 503/
+    )
+  })
+
+  it('rejects a document-library listing that continues beyond its safety limit', async () => {
+    const initialUrl = `${GRAPH}/sites/${SITE_ID}/drives?$select=id,name,webUrl`
+    mockGraph({
+      ...defaultDriveRoute,
+      ...paginatedRoutes(initialUrl, 'drive-pages', 20, true),
+    })
+
+    await expect(resolve('Missing')).rejects.toThrow(
+      /document-library listing exceeded the 20-page safety limit/
+    )
+  })
+
+  it('surfaces a 404 encountered while traversing a folder collection', async () => {
+    mockGraph({ ...defaultDriveRoute, ...sitesDrivesRoute })
+
+    await expect(resolve('Missing')).rejects.toThrow(/Failed to list folder contents: 404/)
+  })
+
+  it('rejects a folder listing that continues beyond its safety limit', async () => {
+    const initialUrl = `${GRAPH}/drives/${DEFAULT_DRIVE_ID}/root/children?$top=200&$select=id,name,folder`
+    mockGraph({
+      ...defaultDriveRoute,
+      ...sitesDrivesRoute,
+      ...paginatedRoutes(initialUrl, 'folder-pages', 50, true),
+    })
+
+    await expect(resolve('Missing')).rejects.toThrow(
+      /folder listing exceeded the 50-page safety limit/
+    )
+  })
+
   it('accepts an address-bar folder URL carrying the path in the id parameter', async () => {
     mockGraph({
       ...defaultDriveRoute,

apps/sim/connectors/sharepoint/sharepoint.ts

@@ -425,9 +425,9 @@ async function listChildFolders(
 
   for (let page = 0; page < MAX_CHILD_PAGES_PER_SEGMENT; page++) {
     const response = await graphGet(url, accessToken, retryOptions)
-    if (response.status === 404) break
     if (!response.ok) {
-      throw new Error(`Failed to list folder contents: ${response.status}`)
+      const errorText = await readBoundedHttpErrorBody(response)
+      throw new Error(`Failed to list folder contents: ${response.status} – ${errorText}`)
     }
 
     const rawData: unknown = await response.json()
@@ -455,11 +455,16 @@ async function listChildFolders(
       rawData['@odata.nextLink'] === undefined
         ? undefined
         : assertMicrosoftGraphNextLink(rawData['@odata.nextLink'])
-    if (!nextLink) break
+    if (!nextLink) return folders
+    if (page === MAX_CHILD_PAGES_PER_SEGMENT - 1) {
+      throw new Error(
+        `SharePoint folder listing exceeded the ${MAX_CHILD_PAGES_PER_SEGMENT}-page safety limit`
+      )
+    }
     url = nextLink
   }
 
-  return folders
+  throw new Error('SharePoint folder listing ended unexpectedly')
 }
 
 /**
@@ -668,15 +673,25 @@ async function listSiteDrives(
 
   for (let page = 0; page < MAX_DRIVE_PAGES; page++) {
     const response = await graphGet(url, accessToken, retryOptions)
-    if (!response.ok) break
+    if (!response.ok) {
+      const errorText = await readBoundedHttpErrorBody(response)
+      throw new Error(
+        `Failed to list SharePoint document libraries: ${response.status} – ${errorText}`
+      )
+    }
     const data = parseDriveListResponse(await response.json())
     drives.push(...data.value)
     const nextLink = data['@odata.nextLink']
-    if (!nextLink) break
+    if (!nextLink) return drives
+    if (page === MAX_DRIVE_PAGES - 1) {
+      throw new Error(
+        `SharePoint document-library listing exceeded the ${MAX_DRIVE_PAGES}-page safety limit`
+      )
+    }
     url = nextLink
   }
 
-  return drives
+  throw new Error('SharePoint document-library listing ended unexpectedly')
 }
 
 /**