fix(knowledge): stop capping the unpaged knowledge-base list

#6770 routed GET /api/knowledge through the paged workspace read, which fails
hard above MAX_KNOWLEDGE_BASES_PER_WORKSPACE. That cap is right for a paged
caller — it can be told the set outgrew its page — but this surface has no cursor
and its callers each want the whole set, so the cap could only ever mean a 500.

Nothing prunes archived knowledge bases, so a long-lived workspace crosses any
fixed row count on its own. Staging did: the first list request after the deploy
returned 500 with "Knowledge base list exceeds the 10000 row limit", against data
that had served fine for days.

Read unbounded here, matching the sibling internal lists (`listTables`,
workspace files), and leave the cap on the paged read where it was designed to
live. The legacy personal read loses its cap for the same reason — it feeds the
same unpaged surface.

Author: waleedlatif1

apps/sim/lib/knowledge/constants.ts

@@ -5,12 +5,6 @@ export const KNOWLEDGE_BASE_DESCRIPTION_MAX_LENGTH = 10_000
 /** Hard bound for full-workspace knowledge-base list projections. */
 export const MAX_KNOWLEDGE_BASES_PER_WORKSPACE = 10_000
 
-/**
- * Cap on one caller's legacy workspace-less knowledge bases. Separate from the per-workspace
- * cap because it bounds a per-user set governed by no workspace rule — the two limits should
- * be free to move independently.
- */
-export const MAX_LEGACY_PERSONAL_KNOWLEDGE_BASES = 10_000
 /** Hard bound for path-indexed knowledge folder trees and recursive cascades. */
 export const MAX_KNOWLEDGE_FOLDERS_PER_WORKSPACE = MAX_FOLDERS_PER_WORKSPACE
 

apps/sim/lib/knowledge/service.test.ts

@@ -110,18 +110,6 @@ describe('getLegacyPersonalKnowledgeBases', () => {
     expect(joinedTables).toContain(schemaMock.document)
     expect(joinedTables).not.toContain(schemaMock.permissions)
   })
-
-  it('fails before projecting connector data for an oversized set', async () => {
-    dbChainMockFns.limit.mockResolvedValueOnce(
-      Array.from({ length: MAX_KNOWLEDGE_BASES_PER_WORKSPACE + 1 }, (_, index) => ({
-        id: `kb-${index}`,
-      }))
-    )
-
-    await expect(getLegacyPersonalKnowledgeBases('user-a')).rejects.toThrow(
-      `Legacy personal knowledge base list exceeds the ${MAX_KNOWLEDGE_BASES_PER_WORKSPACE} row limit`
-    )
-  })
 })
 
 /**
@@ -135,6 +123,27 @@ describe('listWorkspaceAndLegacyKnowledgeBases', () => {
     resetDbChainMock()
   })
 
+  /**
+   * Nothing prunes archived rows, so a long-lived workspace reaches any fixed row cap on its
+   * own. This surface has no cursor to page with, so a cap here can only mean a 500 on the
+   * knowledge page and Recently Deleted — which is exactly what it meant on staging. The cap
+   * belongs to the paged read, where a caller can be told the set outgrew its page.
+   */
+  it('serves a workspace whose archived set is larger than the paged read’s cap', async () => {
+    const rows = Array.from({ length: MAX_KNOWLEDGE_BASES_PER_WORKSPACE + 1 }, (_, index) => ({
+      id: `kb-${index}`,
+      chunkingConfig: {},
+      docCount: 0,
+      createdAt: new Date('2026-01-01T00:00:00Z'),
+    }))
+    dbChainMockFns.orderBy.mockResolvedValueOnce(rows).mockResolvedValueOnce([])
+    dbChainMockFns.limit.mockResolvedValueOnce([])
+
+    const result = await listWorkspaceAndLegacyKnowledgeBases('user-a', 'ws-1', 'archived')
+
+    expect(result).toHaveLength(MAX_KNOWLEDGE_BASES_PER_WORKSPACE + 1)
+  })
+
   it('orders both sources as one list and projects connectors once', async () => {
     const workspaceRow = {
       id: 'kb-workspace',
@@ -148,16 +157,16 @@ describe('listWorkspaceAndLegacyKnowledgeBases', () => {
       docCount: 0,
       createdAt: new Date('2025-01-01T00:00:00Z'),
     }
-    dbChainMockFns.limit
-      .mockResolvedValueOnce([workspaceRow])
-      .mockResolvedValueOnce([legacyRow])
-      .mockResolvedValueOnce([])
+    /** Both row reads are unbounded now, so each resolves at `orderBy`; only the connector
+     *  projection still ends in `limit`. */
+    dbChainMockFns.orderBy.mockResolvedValueOnce([workspaceRow]).mockResolvedValueOnce([legacyRow])
+    dbChainMockFns.limit.mockResolvedValueOnce([])
 
     const result = await listWorkspaceAndLegacyKnowledgeBases('user-a', 'ws-1')
 
     expect(result.map((kb) => kb.id)).toEqual(['kb-legacy', 'kb-workspace'])
-    /** Two row reads and ONE connector projection — three chains, never four. */
-    expect(dbChainMockFns.limit).toHaveBeenCalledTimes(3)
+    /** ONE connector projection over the merged set, not one per source. */
+    expect(dbChainMockFns.limit).toHaveBeenCalledTimes(1)
   })
 })
 

apps/sim/lib/knowledge/service.ts

@@ -31,7 +31,6 @@ import { findActiveFolder, resolveRestoredFolderId } from '@/lib/folders/queries
 import {
   MAX_KNOWLEDGE_BASES_PER_WORKSPACE,
   MAX_KNOWLEDGE_CONNECTOR_TYPE_ROWS_PER_LIST,
-  MAX_LEGACY_PERSONAL_KNOWLEDGE_BASES,
 } from '@/lib/knowledge/constants'
 import type {
   ChunkingConfig,
@@ -181,9 +180,9 @@ function knowledgeBaseScopeCondition(scope: KnowledgeBaseScope) {
 async function readKnowledgeBaseRows(
   where: SQL | undefined,
   orderBy: SQL[],
-  limit: number
+  limit?: number
 ): Promise<Array<Omit<KnowledgeBaseWithCounts, 'connectorTypes'>>> {
-  const rows = await db
+  const query = db
     .select({
       id: knowledgeBase.id,
       userId: knowledgeBase.userId,
@@ -213,7 +212,8 @@ async function readKnowledgeBaseRows(
     .where(where)
     .groupBy(knowledgeBase.id)
     .orderBy(...orderBy)
-    .limit(limit)
+
+  const rows = limit === undefined ? await query : await query.limit(limit)
 
   return rows.map((kb) => ({
     ...kb,
@@ -348,17 +348,9 @@ async function readLegacyPersonalKnowledgeBaseRows(
       eq(knowledgeBase.userId, userId),
       isNull(knowledgeBase.workspaceId)
     ),
-    listOrderBy(keysetColumns(KNOWLEDGE_BASE_SORTS.createdAt), 'asc'),
-    MAX_LEGACY_PERSONAL_KNOWLEDGE_BASES + 1
+    listOrderBy(keysetColumns(KNOWLEDGE_BASE_SORTS.createdAt), 'asc')
   )
 
-  /** One row past the cap, so an oversized set fails loudly instead of truncating in silence. */
-  if (rows.length > MAX_LEGACY_PERSONAL_KNOWLEDGE_BASES) {
-    throw new Error(
-      `Legacy personal knowledge base list exceeds the ${MAX_LEGACY_PERSONAL_KNOWLEDGE_BASES} row limit`
-    )
-  }
-
   return rows
 }
 
@@ -386,8 +378,21 @@ export async function listWorkspaceAndLegacyKnowledgeBases(
   workspaceId: string,
   scope: KnowledgeBaseScope = 'active'
 ): Promise<KnowledgeBaseWithCounts[]> {
+  /**
+   * Unbounded on purpose, matching the sibling internal lists (`listTables`,
+   * `listWorkspaceFiles`): this surface has no cursor to page with and its callers — the
+   * knowledge page, the base selector, Recently Deleted — each want the whole set.
+   *
+   * `MAX_KNOWLEDGE_BASES_PER_WORKSPACE` guards the PAGED read instead, where a caller that
+   * asked for a page can be told the set outgrew it. Applying it here turned a slow list into
+   * a 500 for any workspace that had ever archived more knowledge bases than the cap, which is
+   * a state archived rows reach on their own, since nothing prunes them.
+   */
   const [workspaceRows, legacyPersonalRows] = await Promise.all([
-    readWorkspaceKnowledgeBaseRows(workspaceId, scope).then((page) => page.data),
+    readKnowledgeBaseRows(
+      and(eq(knowledgeBase.workspaceId, workspaceId), knowledgeBaseScopeCondition(scope)),
+      listOrderBy(keysetColumns(KNOWLEDGE_BASE_SORTS.createdAt), 'asc')
+    ),
     readLegacyPersonalKnowledgeBaseRows(userId, scope),
   ])