diff --git a/apps/docs/content/docs/integrations/knowledge.mdx b/apps/docs/content/docs/integrations/knowledge.mdx index f1be279b661..02c044589cb 100644 --- a/apps/docs/content/docs/integrations/knowledge.mdx +++ b/apps/docs/content/docs/integrations/knowledge.mdx @@ -33,7 +33,7 @@ Integrate Knowledge into the workflow. Perform full CRUD operations on documents ### Knowledge Search -Search for similar content in a knowledge base using vector similarity +Search for similar content in a knowledge base by relevance #### Input @@ -43,7 +43,7 @@ Search for similar content in a knowledge base using vector similarity | `query` | string | No | Search query text \(optional when using tag filters\) | | `topK` | number | No | Number of most similar results to return \(1-100\) | | `tagFilters` | array | No | Array of tag filters with tagName and tagValue properties | -| `searchMode` | string | No | Retrieval mode: 'vector' \(default\) uses semantic similarity only, 'hybrid' also runs a full-text leg and fuses both | +| `searchMode` | string | No | Retrieval mode: 'hybrid' fuses a full-text leg with semantic similarity, 'vector' uses semantic similarity only; omit for the workspace's default | | `rerankerEnabled` | boolean | No | Whether to apply Cohere reranking to vector search results | | `rerankerModel` | string | No | Cohere rerank model to use \(one of: rerank-v4.0-pro, rerank-v4.0-fast, rerank-v3.5\) | | `rerankerInputCount` | number | No | Number of vector results sent to the Cohere reranker \(1–100\). Defaults to topK × 4 capped at 100. | diff --git a/apps/docs/openapi-v2-knowledge.json b/apps/docs/openapi-v2-knowledge.json index 15357b66a8b..999ba3d8732 100644 --- a/apps/docs/openapi-v2-knowledge.json +++ b/apps/docs/openapi-v2-knowledge.json @@ -6144,7 +6144,6 @@ }, "searchMode": { "description": "Retrieval strategy: vector is semantic-only, while hybrid also runs full-text search.", - "default": "vector", "anyOf": [ { "type": "string", diff --git a/apps/sim/.env.example b/apps/sim/.env.example index 1c9c6cbd024..90762132025 100644 --- a/apps/sim/.env.example +++ b/apps/sim/.env.example @@ -203,6 +203,7 @@ CRON_SECRET=your_cron_secret # Use `openssl rand -hex 32` to generate. Authentic # FORKING_ENABLED= # Workspace forks # CREDENTIAL_GROUPS= # Enterprise managed OAuth collections # TABLE_ROW_TTL= # Table TTL columns and expired-row cleanup +# KNOWLEDGE_MEMBER_ACCESS= # Per-member knowledge connectors and hybrid-by-default retrieval # ORGANIZATIONS_ENABLED= / NEXT_PUBLIC_ORGANIZATIONS_ENABLED= # Organizations only # Instance organization (Optional). Most enterprise features read their settings from the diff --git a/apps/sim/app/api/files/authorization.ts b/apps/sim/app/api/files/authorization.ts index 77d087bc859..dbac5ed031e 100644 --- a/apps/sim/app/api/files/authorization.ts +++ b/apps/sim/app/api/files/authorization.ts @@ -4,6 +4,12 @@ import { createLogger } from '@sim/logger' import { permissionSatisfies } from '@sim/platform-authz/workspace' import { and, eq, isNull } from 'drizzle-orm' import { NextResponse } from 'next/server' +import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' +import { + resolveUserKnowledgeAccessScope, + WORKSPACE_ACCESS_SCOPE, +} from '@/lib/knowledge/access/scope' +import type { KnowledgeAccessScope, SystemAccessScope } from '@/lib/knowledge/access/types' import { getFileMetadata } from '@/lib/uploads' import type { StorageContext } from '@/lib/uploads/config' import type { StorageConfig } from '@/lib/uploads/core/storage-client' @@ -141,7 +147,7 @@ export async function verifyFileAccess( customConfig?: StorageConfig, context?: StorageContext | 'general', isLocal?: boolean, - options?: { requireWrite?: boolean } + options?: { requireWrite?: boolean; knowledgeAccess?: KnowledgeFileAccess } ): Promise { const requireWrite = options?.requireWrite ?? false try { @@ -182,7 +188,7 @@ export async function verifyFileAccess( // 4. KB files: kb/filename if (inferredContext === 'knowledge-base') { - return await verifyKBFileAccess(cloudKey, userId, customConfig) + return await verifyKBFileAccess(cloudKey, userId, customConfig, options?.knowledgeAccess) } // 5. Chat files: chat/filename @@ -485,7 +491,14 @@ async function verifyCopilotFileAccess( * signal only: it reflects whether the file is still part of a live KB, not who * owns it (ownership comes from the binding). */ -async function hasActiveKbDocumentForKey(cloudKey: string, workspaceId: string): Promise { +/** A reader once resolved: a person's or the workspace's tokens, or the system reading its own rows. */ +type ResolvedKnowledgeFileAccess = KnowledgeAccessScope | SystemAccessScope + +async function hasActiveKbDocumentForKey( + cloudKey: string, + workspaceId: string, + access: ResolvedKnowledgeFileAccess +): Promise { const rows = await db .select({ id: document.id }) .from(document) @@ -497,7 +510,8 @@ async function hasActiveKbDocumentForKey(cloudKey: string, workspaceId: string): eq(document.userExcluded, false), isNull(document.archivedAt), isNull(document.deletedAt), - isNull(knowledgeBase.deletedAt) + isNull(knowledgeBase.deletedAt), + knowledgeAccessCondition(access) ) ) .limit(1) @@ -505,6 +519,26 @@ async function hasActiveKbDocumentForKey(cloudKey: string, workspaceId: string): return rows.length > 0 } +/** + * How a KB file read identifies the reader for document access. `'user'` is + * for a session-authenticated person; a resolved scope is for a caller that + * already holds one (an execution with a principal). The system scope is for + * a background job reading a connector-owned row it is processing, which in + * members mode is hidden until the sync materializes its readers. Anything + * else — an internal token, a tool running with the workflow owner's id — + * reads as the workspace, never as the person whose id it happens to carry. + */ +export type KnowledgeFileAccess = 'user' | ResolvedKnowledgeFileAccess + +async function resolveKnowledgeFileAccess( + knowledgeAccess: KnowledgeFileAccess | undefined, + userId: string, + workspaceId: string +): Promise { + if (knowledgeAccess === 'user') return resolveUserKnowledgeAccessScope(userId, workspaceId) + return knowledgeAccess ?? WORKSPACE_ACCESS_SCOPE +} + /** * Verify access to KB files (`kb/`). * @@ -522,7 +556,8 @@ async function hasActiveKbDocumentForKey(cloudKey: string, workspaceId: string): async function verifyKBFileAccess( cloudKey: string, userId: string, - customConfig?: StorageConfig + customConfig?: StorageConfig, + knowledgeAccess?: KnowledgeFileAccess ): Promise { try { const binding = await getFileMetadataByKey(cloudKey, 'knowledge-base', { @@ -552,10 +587,12 @@ async function verifyKBFileAccess( return false } - if (!(await hasActiveKbDocumentForKey(cloudKey, binding.workspaceId))) { - logger.warn('KB file access denied: no active document references the file', { + const access = await resolveKnowledgeFileAccess(knowledgeAccess, userId, binding.workspaceId) + if (!(await hasActiveKbDocumentForKey(cloudKey, binding.workspaceId, access))) { + logger.warn('KB file access denied: no readable document references the file', { userId, cloudKey, + accessScopeKind: access.kind, }) return false } diff --git a/apps/sim/app/api/files/export/[id]/route.test.ts b/apps/sim/app/api/files/export/[id]/route.test.ts index 7d796a624e8..d28279f83f0 100644 --- a/apps/sim/app/api/files/export/[id]/route.test.ts +++ b/apps/sim/app/api/files/export/[id]/route.test.ts @@ -25,7 +25,10 @@ function embeds(...ids: string[]) { mockExtractEmbeddedFileRefs.mockReturnValue({ keys: [], ids }) } -vi.mock('@/lib/auth/hybrid', () => ({ checkSessionOrInternalAuth: mockCheckAuth })) +vi.mock('@/lib/auth/hybrid', () => ({ + AuthType: { SESSION: 'session', API_KEY: 'api_key', INTERNAL_JWT: 'internal_jwt' }, + checkSessionOrInternalAuth: mockCheckAuth, +})) vi.mock('@/lib/uploads/server/metadata', () => ({ getFileMetadataById: mockGetFileMetadataById, })) diff --git a/apps/sim/app/api/files/export/[id]/route.ts b/apps/sim/app/api/files/export/[id]/route.ts index f36104475aa..1d4b918e692 100644 --- a/apps/sim/app/api/files/export/[id]/route.ts +++ b/apps/sim/app/api/files/export/[id]/route.ts @@ -7,7 +7,7 @@ import type { NextRequest } from 'next/server' import { NextResponse } from 'next/server' import { fileExportContract } from '@/lib/api/contracts/storage-transfer' import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' +import { AuthType, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { MATERIALIZE_CONCURRENCY, mapWithConcurrency } from '@/lib/core/utils/concurrency' import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' @@ -81,7 +81,10 @@ export const GET = withRouteHandler( return NextResponse.json({ error: 'Not found' }, { status: 404 }) } - const hasAccess = await verifyFileAccess(record.key, userId) + const knowledgeAccess = authResult.authType === AuthType.SESSION ? 'user' : undefined + const hasAccess = await verifyFileAccess(record.key, userId, undefined, undefined, undefined, { + knowledgeAccess, + }) if (!hasAccess) { logger.warn('Unauthorized file export attempt', { id, userId }) return NextResponse.json({ error: 'Forbidden' }, { status: 403 }) @@ -164,7 +167,13 @@ export const GET = withRouteHandler( try { const imgRecord = await getFileMetadataById(storedFileId(imageId)) if (!imgRecord) return null - if (!(await verifyFileAccess(imgRecord.key, userId))) return null + if ( + !(await verifyFileAccess(imgRecord.key, userId, undefined, undefined, undefined, { + knowledgeAccess, + })) + ) { + return null + } return { imageId, record: imgRecord, size: getWorkspaceFileSize(imgRecord) } } catch (error) { logger.warn('Failed to resolve asset for export', { diff --git a/apps/sim/app/api/files/serve/[...path]/route.test.ts b/apps/sim/app/api/files/serve/[...path]/route.test.ts index 6bdae38ab55..0adf30b1f5f 100644 --- a/apps/sim/app/api/files/serve/[...path]/route.test.ts +++ b/apps/sim/app/api/files/serve/[...path]/route.test.ts @@ -397,7 +397,8 @@ describe('File Serve API Route', () => { 'test-user-id', undefined, 'mothership', - false + false, + { knowledgeAccess: undefined } ) expect(storageServiceMockFns.mockDownloadFile).toHaveBeenCalledWith({ key: 'workspace/test-workspace-id/1234567890-photo.png', diff --git a/apps/sim/app/api/files/serve/[...path]/route.ts b/apps/sim/app/api/files/serve/[...path]/route.ts index 71fdb72a46f..ffb8845aeff 100644 --- a/apps/sim/app/api/files/serve/[...path]/route.ts +++ b/apps/sim/app/api/files/serve/[...path]/route.ts @@ -8,7 +8,7 @@ import { concealCrossTenantResourceError, InternalUnauthenticatedError, } from '@/lib/api/server/routes' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' +import { AuthType, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { resolveServableDocBytes } from '@/lib/copilot/tools/server/files/doc-compile' import { DocCompileUserError } from '@/lib/copilot/tools/server/files/doc-compile-error' import { asOrchestrationError } from '@/lib/core/orchestration/types' @@ -26,7 +26,7 @@ import { internalWorkspaceFileServeAuth } from '@/lib/workspace-files/api' import { readWorkspaceFileContentByKey } from '@/lib/workspace-files/application/read-workspace-file-content-by-key' import { isSimPageSource, SIM_PAGE_CONTENT_TYPE } from '@/lib/workspace-files/page-compile' import { renderSimPageDocumentWithAssets } from '@/lib/workspace-files/page-document.server' -import { verifyFileAccess } from '@/app/api/files/authorization' +import { type KnowledgeFileAccess, verifyFileAccess } from '@/app/api/files/authorization' import { createErrorResponse, createFileResponse, @@ -271,12 +271,29 @@ export const GET = withRouteHandler( const userId = legacyAuthResult?.userId if (!userId) throw new Error('Authenticated file serve request is missing a user ID') + /** Only a session identifies a person; an internal token's user id reads as the workspace. */ + const knowledgeAccess = + legacyAuthResult?.authType === AuthType.SESSION ? ('user' as const) : undefined if (isUsingCloudStorage()) { - return await handleCloudProxy(cloudKey, userId, options, request.signal, storageContext) + return await handleCloudProxy( + cloudKey, + userId, + options, + request.signal, + storageContext, + knowledgeAccess + ) } - return await handleLocalFile(cloudKey, userId, options, request.signal, storageContext) + return await handleLocalFile( + cloudKey, + userId, + options, + request.signal, + storageContext, + knowledgeAccess + ) } catch (error) { if (error instanceof InternalUnauthenticatedError) { logger.warn('Unauthorized file access attempt', { error: error.message }) @@ -359,7 +376,8 @@ async function handleLocalFile( userId: string, options: ServeOptions, signal: AbortSignal | undefined, - context: StorageContext + context: StorageContext, + knowledgeAccess: KnowledgeFileAccess | undefined ): Promise { const ownerKey = `user:${userId}` try { @@ -368,7 +386,8 @@ async function handleLocalFile( userId, undefined, // customConfig context, - true // isLocal + true, // isLocal + { knowledgeAccess } ) if (!hasAccess) { @@ -419,7 +438,8 @@ async function handleCloudProxy( userId: string, options: ServeOptions, signal: AbortSignal | undefined, - context: StorageContext + context: StorageContext, + knowledgeAccess: KnowledgeFileAccess | undefined ): Promise { const ownerKey = `user:${userId}` try { @@ -430,7 +450,8 @@ async function handleCloudProxy( userId, undefined, // customConfig context, // context - false // isLocal + false, // isLocal + { knowledgeAccess } ) if (!hasAccess) { diff --git a/apps/sim/app/api/files/view/[id]/route.ts b/apps/sim/app/api/files/view/[id]/route.ts index 55ddedcef3a..47ab06d164f 100644 --- a/apps/sim/app/api/files/view/[id]/route.ts +++ b/apps/sim/app/api/files/view/[id]/route.ts @@ -3,7 +3,7 @@ import type { NextRequest } from 'next/server' import { NextResponse } from 'next/server' import { fileViewContract } from '@/lib/api/contracts/storage-transfer' import { parseRequest } from '@/lib/api/server' -import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid' +import { AuthType, checkSessionOrInternalAuth } from '@/lib/auth/hybrid' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { getServeStoragePrefix, type StorageContext } from '@/lib/uploads/config' import { getFileMetadataById } from '@/lib/uploads/server/metadata' @@ -37,7 +37,9 @@ export const GET = withRouteHandler( record.key, authResult.userId, undefined, - record.context as StorageContext | 'general' + record.context as StorageContext | 'general', + undefined, + { knowledgeAccess: authResult.authType === AuthType.SESSION ? 'user' : undefined } ) if (!hasAccess) { logger.warn('Unauthorized file view attempt', { id, userId: authResult.userId }) diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/access/route.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/access/route.ts new file mode 100644 index 00000000000..ab89571ab7e --- /dev/null +++ b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/access/route.ts @@ -0,0 +1,39 @@ +import { updateKnowledgeConnectorAccessContract } from '@/lib/api/contracts/knowledge' +import { + defineInternalJsonRoute, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { + resolveInternalKnowledgeBillingAttribution, + toInternalKnowledgeConnector, +} from '@/lib/knowledge/api/internal-route' +import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' +import { updateKnowledgeConnectorAccess } from '@/lib/knowledge/application/connector-access' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' + +export const PATCH = defineInternalJsonRoute({ + contract: updateKnowledgeConnectorAccessContract, + auth: internalSessionAuth, + operation: knowledgeOperations.updateConnectorAccess, + rateLimit: internalRateLimits.none({ + reason: 'A settings action an admin performs by hand; the switch itself is bounded', + }), + errorPolicy: internalKnowledgeErrorPolicies.connectors, + mapInput: ({ params, body }, { principal, request }) => ({ + connectorId: params.connectorId, + knowledgeBaseId: params.id, + accessMode: body.accessMode, + credentialGroupId: body.credentialGroupId, + credentialGroupOptionId: body.credentialGroupOptionId, + credentialId: body.credentialId, + resolveBillingAttribution: (workspaceId: string) => + resolveInternalKnowledgeBillingAttribution(request, principal, workspaceId), + source: 'ui' as const, + }), + useCase: updateKnowledgeConnectorAccess, + present: ({ connector }) => ({ + success: true as const, + data: toInternalKnowledgeConnector(connector), + }), +}) diff --git a/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/enroll/route.ts b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/enroll/route.ts new file mode 100644 index 00000000000..da3cc91c176 --- /dev/null +++ b/apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/enroll/route.ts @@ -0,0 +1,26 @@ +import { startKnowledgeConnectorMemberEnrollmentContract } from '@/lib/api/contracts/knowledge' +import { + defineInternalJsonRoute, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' +import { startKnowledgeConnectorMemberEnrollment } from '@/lib/knowledge/application/connector-access' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' + +export const POST = defineInternalJsonRoute({ + contract: startKnowledgeConnectorMemberEnrollmentContract, + auth: internalSessionAuth, + operation: knowledgeOperations.enrollConnectorMember, + rateLimit: internalRateLimits.none({ + reason: + 'A member connecting their own account by hand; each call only re-issues their own invitation', + }), + errorPolicy: internalKnowledgeErrorPolicies.connectors, + mapInput: ({ params }) => ({ + connectorId: params.connectorId, + knowledgeBaseId: params.id, + }), + useCase: startKnowledgeConnectorMemberEnrollment, + present: ({ url }) => ({ success: true as const, data: { url } }), +}) diff --git a/apps/sim/app/api/knowledge/[id]/connectors/route.ts b/apps/sim/app/api/knowledge/[id]/connectors/route.ts index b28042f6481..28199775a22 100644 --- a/apps/sim/app/api/knowledge/[id]/connectors/route.ts +++ b/apps/sim/app/api/knowledge/[id]/connectors/route.ts @@ -49,6 +49,9 @@ export const POST = defineInternalJsonRoute({ apiKey: body.apiKey, sourceConfig: body.sourceConfig, syncIntervalMinutes: body.syncIntervalMinutes, + accessMode: body.accessMode, + credentialGroupId: body.credentialGroupId, + credentialGroupOptionId: body.credentialGroupOptionId, resolveBillingAttribution: (workspaceId: string) => resolveInternalKnowledgeBillingAttribution(request, principal, workspaceId), source: 'ui' as const, diff --git a/apps/sim/app/api/knowledge/connectors/member-sync/route.ts b/apps/sim/app/api/knowledge/connectors/member-sync/route.ts new file mode 100644 index 00000000000..076344f81dd --- /dev/null +++ b/apps/sim/app/api/knowledge/connectors/member-sync/route.ts @@ -0,0 +1,221 @@ +import { db } from '@sim/db' +import { knowledgeBase, knowledgeConnector, knowledgeConnectorMemberSyncLog } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { and, asc, eq, inArray, isNull, lte, type SQL, sql } from 'drizzle-orm' +import { type NextRequest, NextResponse } from 'next/server' +import { verifyCronAuth } from '@/lib/auth/internal' +import { resolveSystemBillingAttribution } from '@/lib/billing/core/billing-attribution' +import { mapWithConcurrency } from '@/lib/core/utils/concurrency' +import { generateRequestId } from '@/lib/core/utils/request' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { sweepStaleMemberObservations } from '@/lib/knowledge/connectors/member-observations' +import { + dispatchMemberSync, + QUEUEABLE_MEMBER_SYNC_STATUSES, +} from '@/lib/knowledge/connectors/member-queue' +import { + CONNECTOR_AUTO_DISABLED_ERROR, + CONNECTOR_FAILURE_BACKOFF_CAP_MINUTES, + CONNECTOR_FAILURE_BACKOFF_STEP_MINUTES, + MAX_CONSECUTIVE_FAILURES, + MEMBER_SYNC_STALE_LOCK_TTL_MS, +} from '@/lib/knowledge/connectors/sync-limits' + +export const dynamic = 'force-dynamic' + +const logger = createLogger('ConnectorMemberSyncSchedulerAPI') + +const MAX_DISPATCHES_PER_TICK = 200 +const DISPATCH_CONCURRENCY = 10 +const STALE_LOCK_ERROR_MESSAGE = 'Member sync timed out (stale lock recovered)' +const LOST_DISPATCH_ERROR_MESSAGE = 'Member sync was queued but never started' + +/** The member lease, read through `COALESCE` for the same reason the content reaper does. */ +function memberSyncLockLease(): SQL { + return sql`COALESCE(${knowledgeConnector.memberSyncLockLeaseAt}, ${knowledgeConnector.updatedAt})` +} + +function reclaimedFailureCount(): SQL { + return sql`COALESCE(${knowledgeConnector.memberSyncConsecutiveFailures}, 0) + 1` +} + +function reclaimedStatus(): SQL { + return sql`CASE WHEN ${reclaimedFailureCount()} >= ${MAX_CONSECUTIVE_FAILURES} THEN 'disabled' ELSE 'error' END` +} + +function reclaimedError(message: string): SQL { + return sql`CASE WHEN ${reclaimedFailureCount()} >= ${MAX_CONSECUTIVE_FAILURES} THEN ${CONNECTOR_AUTO_DISABLED_ERROR} ELSE ${message} END` +} + +function reclaimedNextMemberSyncAt(): SQL { + return sql`CASE WHEN ${reclaimedFailureCount()} >= ${MAX_CONSECUTIVE_FAILURES} THEN NULL ELSE now() + LEAST(${reclaimedFailureCount()} * ${CONNECTOR_FAILURE_BACKOFF_STEP_MINUTES}, ${CONNECTOR_FAILURE_BACKOFF_CAP_MINUTES}) * INTERVAL '1 minute' END` +} + +/** + * The write shared by both reclaims: a run that stopped making progress + * re-enters the member failure ladder, which is the content engine's ladder + * over the member columns. + */ +function reclaimPayload(message: string) { + return { + memberSyncStatus: reclaimedStatus(), + lastMemberSyncError: reclaimedError(message), + nextMemberSyncAt: reclaimedNextMemberSyncAt(), + memberSyncConsecutiveFailures: reclaimedFailureCount(), + memberSyncLockToken: null, + memberSyncLockLeaseAt: null, + updatedAt: sql`now()`, + } +} + +/** Spares a member-sync log row whose run is provably still heartbeating. */ +function logRowNotHeldByLiveRun(staleCutoff: Date): SQL { + return sql`NOT EXISTS ( + SELECT 1 FROM ${knowledgeConnector} + WHERE ${knowledgeConnector.id} = ${knowledgeConnectorMemberSyncLog.connectorId} + AND ${knowledgeConnector.memberSyncLockToken} = ${knowledgeConnectorMemberSyncLog.id} + AND ${knowledgeConnector.memberSyncStatus} = 'running' + AND ${memberSyncLockLease()} > ${sql.param(staleCutoff, knowledgeConnector.memberSyncLockLeaseAt)} + )` +} + +/** + * Cron endpoint for members-mode connectors: reclaims stale leases and lost + * queue entries, closes orphaned run logs, sweeps members whose crawls + * stopped, and dispatches every connector that is due. Runs every 5 minutes. + */ +export const GET = withRouteHandler(async (request: NextRequest) => { + const requestId = generateRequestId() + logger.info(`[${requestId}] Connector member sync scheduler triggered`) + + const authError = verifyCronAuth(request, 'Connector member sync scheduler') + if (authError) return authError + + try { + const now = new Date() + const staleCutoff = new Date(now.getTime() - MEMBER_SYNC_STALE_LOCK_TTL_MS) + + const [recoveredRunning, recoveredPending, closedLogs] = await Promise.all([ + db + .update(knowledgeConnector) + .set(reclaimPayload(STALE_LOCK_ERROR_MESSAGE)) + .where( + and( + eq(knowledgeConnector.memberSyncStatus, 'running'), + sql`${memberSyncLockLease()} <= ${sql.param(staleCutoff, knowledgeConnector.memberSyncLockLeaseAt)}`, + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + .returning({ id: knowledgeConnector.id }), + db + .update(knowledgeConnector) + .set(reclaimPayload(LOST_DISPATCH_ERROR_MESSAGE)) + .where( + and( + eq(knowledgeConnector.memberSyncStatus, 'pending'), + sql`${memberSyncLockLease()} <= ${sql.param(staleCutoff, knowledgeConnector.memberSyncLockLeaseAt)}`, + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + .returning({ id: knowledgeConnector.id }), + db + .update(knowledgeConnectorMemberSyncLog) + .set({ + status: 'failed', + completedAt: sql`now()`, + errorMessage: STALE_LOCK_ERROR_MESSAGE, + }) + .where( + and( + eq(knowledgeConnectorMemberSyncLog.status, 'started'), + lte(knowledgeConnectorMemberSyncLog.startedAt, staleCutoff), + logRowNotHeldByLiveRun(staleCutoff) + ) + ) + .returning({ id: knowledgeConnectorMemberSyncLog.id }), + ]) + + if (recoveredRunning.length > 0) { + logger.warn(`[${requestId}] Recovered ${recoveredRunning.length} stale member sync run(s)`, { + ids: recoveredRunning.map((row) => row.id), + }) + } + if (recoveredPending.length > 0) { + logger.warn( + `[${requestId}] Recovered ${recoveredPending.length} connector(s) whose queued member sync never started`, + { ids: recoveredPending.map((row) => row.id) } + ) + } + if (closedLogs.length > 0) { + logger.warn(`[${requestId}] Closed ${closedLogs.length} orphaned member sync log(s)`) + } + + const sweep = await sweepStaleMemberObservations(now) + if (sweep.members > 0) { + logger.warn(`[${requestId}] Swept observations of ${sweep.members} stale member(s)`, sweep) + } + + const dueConnectors = await db + .select({ + id: knowledgeConnector.id, + nextMemberSyncAt: knowledgeConnector.nextMemberSyncAt, + workspaceId: knowledgeBase.workspaceId, + }) + .from(knowledgeConnector) + .innerJoin(knowledgeBase, eq(knowledgeConnector.knowledgeBaseId, knowledgeBase.id)) + .where( + and( + eq(knowledgeConnector.accessMode, 'members'), + inArray(knowledgeConnector.status, ['active', 'error']), + inArray(knowledgeConnector.memberSyncStatus, QUEUEABLE_MEMBER_SYNC_STATUSES), + lte(knowledgeConnector.nextMemberSyncAt, now), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt), + isNull(knowledgeBase.deletedAt) + ) + ) + .orderBy(asc(knowledgeConnector.nextMemberSyncAt)) + .limit(MAX_DISPATCHES_PER_TICK) + + logger.info(`[${requestId}] Found ${dueConnectors.length} connectors due for member sync`) + + if (dueConnectors.length === 0) { + return NextResponse.json({ + success: true, + message: 'No connectors due for member sync', + count: 0, + }) + } + + await mapWithConcurrency(dueConnectors, DISPATCH_CONCURRENCY, async (connector) => { + try { + if (!connector.workspaceId) { + throw new Error(`Connector ${connector.id} is missing workspace billing context`) + } + const billingAttribution = await resolveSystemBillingAttribution(connector.workspaceId) + await dispatchMemberSync(connector.id, { + billingAttribution, + expectedNextMemberSyncAt: connector.nextMemberSyncAt ?? undefined, + requestId, + requireRunnable: true, + }) + } catch (error) { + logger.error( + `[${requestId}] Failed to dispatch member sync for connector ${connector.id}`, + error + ) + } + }) + + return NextResponse.json({ + success: true, + message: `Dispatched ${dueConnectors.length} member sync(s)`, + count: dueConnectors.length, + }) + } catch (error) { + logger.error(`[${requestId}] Connector member sync scheduler error`, error) + return NextResponse.json({ error: 'Internal server error' }, { status: 500 }) + } +}) diff --git a/apps/sim/app/api/knowledge/connectors/sync/route.ts b/apps/sim/app/api/knowledge/connectors/sync/route.ts index 4c98bbaf197..7d3d5afcbd6 100644 --- a/apps/sim/app/api/knowledge/connectors/sync/route.ts +++ b/apps/sim/app/api/knowledge/connectors/sync/route.ts @@ -304,6 +304,7 @@ export const GET = withRouteHandler(async (request: NextRequest) => { .where( and( inArray(knowledgeConnector.status, ['active', 'error']), + eq(knowledgeConnector.accessMode, 'workspace'), lte(knowledgeConnector.nextSyncAt, now), isNull(knowledgeConnector.archivedAt), isNull(knowledgeConnector.deletedAt), diff --git a/apps/sim/app/api/knowledge/member-connectors/route.ts b/apps/sim/app/api/knowledge/member-connectors/route.ts new file mode 100644 index 00000000000..4467c3ec35e --- /dev/null +++ b/apps/sim/app/api/knowledge/member-connectors/route.ts @@ -0,0 +1,20 @@ +import { listWorkspaceMemberConnectorsContract } from '@/lib/api/contracts/knowledge' +import { + defineInternalJsonRoute, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' +import { listWorkspaceMemberConnectors } from '@/lib/knowledge/application/connectors' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' + +export const GET = defineInternalJsonRoute({ + contract: listWorkspaceMemberConnectorsContract, + auth: internalSessionAuth, + operation: knowledgeOperations.listWorkspaceMemberConnectors, + rateLimit: internalRateLimits.none({ reason: 'One small read per visit to the Search tab' }), + errorPolicy: internalKnowledgeErrorPolicies.connectors, + mapInput: ({ query }) => ({ workspaceId: query.workspaceId }), + useCase: listWorkspaceMemberConnectors, + present: ({ connectors }) => ({ success: true as const, data: connectors }), +}) diff --git a/apps/sim/app/api/knowledge/search/route.ts b/apps/sim/app/api/knowledge/search/route.ts new file mode 100644 index 00000000000..96d78f3d1d7 --- /dev/null +++ b/apps/sim/app/api/knowledge/search/route.ts @@ -0,0 +1,49 @@ +import { searchWorkspaceKnowledgeContract } from '@/lib/api/contracts/knowledge' +import { + defineInternalJsonRoute, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { searchKnowledge } from '@/lib/knowledge/application/search' +import { sourceAuthor } from '@/lib/knowledge/search/author' + +export const POST = defineInternalJsonRoute({ + contract: searchWorkspaceKnowledgeContract, + auth: internalSessionAuth, + operation: knowledgeOperations.search, + rateLimit: internalRateLimits.none({ + reason: 'A person typing queries; the embedding call is metered against their workspace', + }), + errorPolicy: internalKnowledgeErrorPolicies.search, + mapInput: ({ body }) => ({ + workspaceId: body.workspaceId, + knowledgeBaseIds: body.knowledgeBaseIds, + query: body.query, + topK: body.topK, + }), + useCase: searchKnowledge, + present: ({ results, knowledgeBases }, { input }) => { + const knowledgeBaseNames = new Map(knowledgeBases.map((kb) => [kb.id, kb.name])) + return { + success: true as const, + data: { + query: input.query ?? '', + results: results.map((result) => ({ + documentId: result.documentId, + knowledgeBaseId: result.knowledgeBaseId, + knowledgeBaseName: knowledgeBaseNames.get(result.knowledgeBaseId) ?? '', + documentName: result.documentName, + sourceUrl: result.sourceUrl, + connectorType: result.connectorType, + sourceModifiedAt: result.sourceModifiedAt?.toISOString() ?? null, + author: sourceAuthor(result.metadata), + content: result.content, + chunkIndex: result.chunkIndex, + similarity: result.similarity, + })), + }, + } + }, +}) diff --git a/apps/sim/app/api/knowledge/search/utils.test.ts b/apps/sim/app/api/knowledge/search/utils.test.ts index a14e55dcf3b..169aac67c7d 100644 --- a/apps/sim/app/api/knowledge/search/utils.test.ts +++ b/apps/sim/app/api/knowledge/search/utils.test.ts @@ -47,18 +47,19 @@ afterEach(() => { Object.assign(env, envSnapshot) }) +import { WORKSPACE_ACCESS_SCOPE } from '@/lib/knowledge/access/scope' +import { generateSearchEmbedding } from '@/lib/knowledge/embeddings' import { executeKeywordSearch, executeKnowledgeSearch, fuseByReciprocalRank, - generateSearchEmbedding, getQueryStrategy, handleTagAndVectorSearch, handleTagOnlySearch, handleVectorOnlySearch, - RRF_K, type SearchResult, } from '@/lib/knowledge/search/queries' +import { RRF_K } from '@/lib/knowledge/search/recency' /** Minimal SearchResult builder — only the fields fusion and ordering read. */ function makeResult(id: string, distance = 0.1): SearchResult { @@ -117,6 +118,7 @@ describe('Knowledge Search Utils', () => { it('should throw error when no filters provided', async () => { const params = { knowledgeBaseIds: ['kb-123'], + access: WORKSPACE_ACCESS_SCOPE, topK: 10, structuredFilters: [], } @@ -129,6 +131,7 @@ describe('Knowledge Search Utils', () => { it('should accept valid parameters for tag-only search', async () => { const params = { knowledgeBaseIds: ['kb-123'], + access: WORKSPACE_ACCESS_SCOPE, topK: 10, structuredFilters: [{ tagSlot: 'tag1', fieldType: 'text', operator: 'eq', value: 'api' }], } @@ -145,6 +148,7 @@ describe('Knowledge Search Utils', () => { it('should throw error when queryVector not provided', async () => { const params = { knowledgeBaseIds: ['kb-123'], + access: WORKSPACE_ACCESS_SCOPE, topK: 10, distanceThreshold: 0.8, } @@ -157,6 +161,7 @@ describe('Knowledge Search Utils', () => { it('should throw error when distanceThreshold not provided', async () => { const params = { knowledgeBaseIds: ['kb-123'], + access: WORKSPACE_ACCESS_SCOPE, topK: 10, queryVector: JSON.stringify([0.1, 0.2, 0.3]), } @@ -169,6 +174,7 @@ describe('Knowledge Search Utils', () => { it('should accept valid parameters for vector-only search', async () => { const params = { knowledgeBaseIds: ['kb-123'], + access: WORKSPACE_ACCESS_SCOPE, topK: 10, queryVector: JSON.stringify([0.1, 0.2, 0.3]), distanceThreshold: 0.8, @@ -186,6 +192,7 @@ describe('Knowledge Search Utils', () => { it('should throw error when no filters provided', async () => { const params = { knowledgeBaseIds: ['kb-123'], + access: WORKSPACE_ACCESS_SCOPE, topK: 10, structuredFilters: [], queryVector: JSON.stringify([0.1, 0.2, 0.3]), @@ -200,6 +207,7 @@ describe('Knowledge Search Utils', () => { it('should throw error when queryVector not provided', async () => { const params = { knowledgeBaseIds: ['kb-123'], + access: WORKSPACE_ACCESS_SCOPE, topK: 10, structuredFilters: [{ tagSlot: 'tag1', fieldType: 'text', operator: 'eq', value: 'api' }], distanceThreshold: 0.8, @@ -213,6 +221,7 @@ describe('Knowledge Search Utils', () => { it('should throw error when distanceThreshold not provided', async () => { const params = { knowledgeBaseIds: ['kb-123'], + access: WORKSPACE_ACCESS_SCOPE, topK: 10, structuredFilters: [{ tagSlot: 'tag1', fieldType: 'text', operator: 'eq', value: 'api' }], queryVector: JSON.stringify([0.1, 0.2, 0.3]), @@ -226,6 +235,7 @@ describe('Knowledge Search Utils', () => { it('should accept valid parameters for tag and vector search', async () => { const params = { knowledgeBaseIds: ['kb-123'], + access: WORKSPACE_ACCESS_SCOPE, topK: 10, structuredFilters: [{ tagSlot: 'tag1', fieldType: 'text', operator: 'eq', value: 'api' }], queryVector: JSON.stringify([0.1, 0.2, 0.3]), @@ -382,6 +392,7 @@ describe('Knowledge Search Utils', () => { it('returns nothing for a whitespace-only query without touching the database', async () => { const results = await executeKeywordSearch({ knowledgeBaseIds: ['kb-123'], + access: WORKSPACE_ACCESS_SCOPE, topK: 10, query: ' ', queryVector: JSON.stringify([0.1, 0.2, 0.3]), @@ -397,6 +408,7 @@ describe('Knowledge Search Utils', () => { await executeKeywordSearch({ knowledgeBaseIds, + access: WORKSPACE_ACCESS_SCOPE, topK: 10, query: 'PROJ-1234', queryVector: JSON.stringify([0.1, 0.2, 0.3]), @@ -416,6 +428,7 @@ describe('Knowledge Search Utils', () => { const results = await executeKeywordSearch({ knowledgeBaseIds: ['kb-1'], + access: WORKSPACE_ACCESS_SCOPE, topK: 10, query: 'PROJ-1234', queryVector: JSON.stringify([0.1, 0.2, 0.3]), @@ -441,6 +454,7 @@ describe('Knowledge Search Utils', () => { await executeKeywordSearch({ knowledgeBaseIds, + access: WORKSPACE_ACCESS_SCOPE, topK: 10, query: 'PROJ-1234', queryVector: JSON.stringify([0.1, 0.2, 0.3]), @@ -459,6 +473,7 @@ describe('Knowledge Search Utils', () => { await expect( executeKnowledgeSearch({ knowledgeBaseIds: ['kb-123'], + access: WORKSPACE_ACCESS_SCOPE, topK: 10, searchMode: 'hybrid', }) @@ -469,6 +484,7 @@ describe('Knowledge Search Utils', () => { await expect( executeKnowledgeSearch({ knowledgeBaseIds: ['kb-123'], + access: WORKSPACE_ACCESS_SCOPE, topK: 10, searchMode: 'hybrid', query: 'PROJ-1234', @@ -481,6 +497,7 @@ describe('Knowledge Search Utils', () => { const results = await executeKnowledgeSearch({ knowledgeBaseIds: ['kb-123'], + access: WORKSPACE_ACCESS_SCOPE, topK: 10, searchMode: 'vector', query: 'PROJ-1234', @@ -492,13 +509,19 @@ describe('Knowledge Search Utils', () => { }) it('runs both legs and fuses them in hybrid mode', async () => { - // Vector leg, then the keyword leg's ranking pass, then its hydration pass. - queueTableRows(schemaMock.embedding, [makeResult('vector-hit')]) + /** + * Chains dequeue in creation order. Hybrid legs over-fetch past the + * plain scan's candidate pool, so the vector leg opens its transaction + * and applies the scan settings before selecting: the keyword ranking + * pass is built first, then the vector select, then hydration. + */ queueTableRows(schemaMock.embedding, [{ id: 'keyword-hit', keywordRank: 0.9 }]) + queueTableRows(schemaMock.embedding, [makeResult('vector-hit')]) queueTableRows(schemaMock.embedding, [makeResult('keyword-hit')]) const results = await executeKnowledgeSearch({ knowledgeBaseIds: ['kb-123'], + access: WORKSPACE_ACCESS_SCOPE, topK: 10, searchMode: 'hybrid', query: 'PROJ-1234', @@ -510,6 +533,8 @@ describe('Knowledge Search Utils', () => { }) it('falls back to vector results when the keyword leg fails', async () => { + /** The failing ranking chain is still built first and takes the first queued set. */ + queueTableRows(schemaMock.embedding, [{ id: 'never-ranked', keywordRank: 0 }]) queueTableRows(schemaMock.embedding, [makeResult('vector-hit')]) /** @@ -529,6 +554,7 @@ describe('Knowledge Search Utils', () => { const results = await executeKnowledgeSearch({ knowledgeBaseIds: ['kb-123'], + access: WORKSPACE_ACCESS_SCOPE, topK: 10, searchMode: 'hybrid', query: 'PROJ-1234', @@ -543,6 +569,7 @@ describe('Knowledge Search Utils', () => { const results = await executeKnowledgeSearch({ knowledgeBaseIds: ['kb-123'], + access: WORKSPACE_ACCESS_SCOPE, topK: 10, searchMode: 'hybrid', structuredFilters: [ diff --git a/apps/sim/app/api/knowledge/sim-search/connect/route.ts b/apps/sim/app/api/knowledge/sim-search/connect/route.ts new file mode 100644 index 00000000000..c13ebd49c16 --- /dev/null +++ b/apps/sim/app/api/knowledge/sim-search/connect/route.ts @@ -0,0 +1,24 @@ +import { connectSimSearchConnectorContract } from '@/lib/api/contracts/knowledge' +import { + defineInternalJsonRoute, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { internalKnowledgeErrorPolicies } from '@/lib/knowledge/api/route-policies' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { connectSimSearchConnector } from '@/lib/knowledge/application/sim-search' + +export const POST = defineInternalJsonRoute({ + contract: connectSimSearchConnectorContract, + auth: internalSessionAuth, + operation: knowledgeOperations.simSearchConnect, + rateLimit: internalRateLimits.none({ reason: 'One click per source; mints a single-use link' }), + errorPolicy: internalKnowledgeErrorPolicies.connectors, + mapInput: ({ body }) => ({ + workspaceId: body.workspaceId, + connectorType: body.connectorType, + sourceConfig: body.sourceConfig, + }), + useCase: connectSimSearchConnector, + present: (result) => ({ success: true as const, data: result }), +}) diff --git a/apps/sim/app/api/knowledge/utils.test.ts b/apps/sim/app/api/knowledge/utils.test.ts index c1a6b445904..d20f5c8b4a2 100644 --- a/apps/sim/app/api/knowledge/utils.test.ts +++ b/apps/sim/app/api/knowledge/utils.test.ts @@ -134,11 +134,7 @@ vi.stubGlobal('fetch', createEmbeddingFetchMock()) import { processDocumentAsync } from '@/lib/knowledge/documents/service' import { generateEmbeddings } from '@/lib/knowledge/embeddings' -import { - checkChunkAccess, - checkDocumentAccess, - checkKnowledgeBaseAccess, -} from '@/app/api/knowledge/utils' +import { checkKnowledgeBaseAccess } from '@/app/api/knowledge/utils' describe('Knowledge Utils', () => { beforeEach(() => { @@ -250,49 +246,6 @@ describe('Knowledge Utils', () => { }) }) - describe('checkDocumentAccess', () => { - it('should return unauthorized when user mismatch', async () => { - queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb1', userId: 'owner' }]) - const result = await checkDocumentAccess('kb1', 'doc1', 'intruder') - - expect(result.hasAccess).toBe(false) - if ('reason' in result) { - expect(result.reason).toBe('Unauthorized knowledge base access') - } - }) - }) - - describe('checkChunkAccess', () => { - it('should fail when document is not completed', async () => { - queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb1', userId: 'user1' }]) - queueTableRows(schemaMock.document, [ - { id: 'doc1', knowledgeBaseId: 'kb1', processingStatus: 'processing' }, - ]) - - const result = await checkChunkAccess('kb1', 'doc1', 'chunk1', 'user1') - - expect(result.hasAccess).toBe(false) - if ('reason' in result) { - expect(result.reason).toContain('Document is not ready') - } - }) - - it('should return success for valid access', async () => { - queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb1', userId: 'user1' }]) - queueTableRows(schemaMock.document, [ - { id: 'doc1', knowledgeBaseId: 'kb1', processingStatus: 'completed' }, - ]) - queueTableRows(schemaMock.embedding, [{ id: 'chunk1', documentId: 'doc1' }]) - - const result = await checkChunkAccess('kb1', 'doc1', 'chunk1', 'user1') - - expect(result.hasAccess).toBe(true) - if ('chunk' in result) { - expect(result.chunk.id).toBe('chunk1') - } - }) - }) - describe('generateEmbeddings', () => { it('should return same length as input', async () => { const result = await generateEmbeddings(['a', 'b']) diff --git a/apps/sim/app/api/knowledge/utils.ts b/apps/sim/app/api/knowledge/utils.ts index e92dc49f419..20a00d38cba 100644 --- a/apps/sim/app/api/knowledge/utils.ts +++ b/apps/sim/app/api/knowledge/utils.ts @@ -1,7 +1,6 @@ import { db } from '@sim/db' -import { embedding, knowledgeBase } from '@sim/db/schema' +import { knowledgeBase } from '@sim/db/schema' import { and, eq, isNull } from 'drizzle-orm' -import { getKnowledgeDocument } from '@/lib/knowledge/documents/service' import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils' interface KnowledgeBaseData { @@ -19,89 +18,6 @@ interface KnowledgeBaseData { updatedAt: Date } -interface DocumentData { - id: string - knowledgeBaseId: string - filename: string - fileUrl: string - fileSize: number - mimeType: string - chunkCount: number - tokenCount: number - characterCount: number - processingStatus: string - processingStartedAt?: Date | null - processingCompletedAt?: Date | null - processingError?: string | null - enabled: boolean - deletedAt?: Date | null - uploadedAt: Date - // Text tags - tag1?: string | null - tag2?: string | null - tag3?: string | null - tag4?: string | null - tag5?: string | null - tag6?: string | null - tag7?: string | null - // Number tags (5 slots) - number1?: number | null - number2?: number | null - number3?: number | null - number4?: number | null - number5?: number | null - // Date tags (2 slots) - date1?: Date | null - date2?: Date | null - // Boolean tags (3 slots) - boolean1?: boolean | null - boolean2?: boolean | null - boolean3?: boolean | null - // Connector fields - connectorId?: string | null - sourceUrl?: string | null - externalId?: string | null -} - -interface EmbeddingData { - id: string - knowledgeBaseId: string - documentId: string - chunkIndex: number - chunkHash: string - content: string - contentLength: number - tokenCount: number - embedding?: number[] | null - embeddingModel: string - startOffset: number - endOffset: number - // Text tags - tag1?: string | null - tag2?: string | null - tag3?: string | null - tag4?: string | null - tag5?: string | null - tag6?: string | null - tag7?: string | null - // Number tags (5 slots) - number1?: number | null - number2?: number | null - number3?: number | null - number4?: number | null - number5?: number | null - // Date tags (2 slots) - date1?: Date | null - date2?: Date | null - // Boolean tags (3 slots) - boolean1?: boolean | null - boolean2?: boolean | null - boolean3?: boolean | null - enabled: boolean - createdAt: Date - updatedAt: Date -} - export interface KnowledgeBaseAccessResult { hasAccess: true knowledgeBase: Pick< @@ -118,41 +34,6 @@ interface KnowledgeBaseAccessDenied { export type KnowledgeBaseAccessCheck = KnowledgeBaseAccessResult | KnowledgeBaseAccessDenied -interface DocumentAccessResult { - hasAccess: true - document: DocumentData - knowledgeBase: Pick< - KnowledgeBaseData, - 'id' | 'userId' | 'workspaceId' | 'name' | 'embeddingModel' - > -} - -interface DocumentAccessDenied { - hasAccess: false - notFound?: boolean - reason: string -} - -export type DocumentAccessCheck = DocumentAccessResult | DocumentAccessDenied - -interface ChunkAccessResult { - hasAccess: true - chunk: EmbeddingData - document: DocumentData - knowledgeBase: Pick< - KnowledgeBaseData, - 'id' | 'userId' | 'workspaceId' | 'name' | 'embeddingModel' - > -} - -interface ChunkAccessDenied { - hasAccess: false - notFound?: boolean - reason: string -} - -export type ChunkAccessCheck = ChunkAccessResult | ChunkAccessDenied - /** * Resolve knowledge-base access for a user, gated by read or write permission. * @@ -223,142 +104,3 @@ export async function checkKnowledgeBaseWriteAccess( ): Promise { return resolveKnowledgeBaseAccess(knowledgeBaseId, userId, true) } - -/** - * Resolve document access within a knowledge base, gated by read or write - * permission on the KB (see {@link resolveKnowledgeBaseAccess}). - */ -async function resolveDocumentAccess( - knowledgeBaseId: string, - documentId: string, - userId: string, - requireWrite: boolean -): Promise { - const kbAccess = await resolveKnowledgeBaseAccess(knowledgeBaseId, userId, requireWrite) - - if (!kbAccess.hasAccess) { - return { - hasAccess: false, - notFound: kbAccess.notFound, - reason: kbAccess.notFound ? 'Knowledge base not found' : 'Unauthorized knowledge base access', - } - } - - const doc = await getKnowledgeDocument(knowledgeBaseId, documentId) - if (!doc) { - return { hasAccess: false, notFound: true, reason: 'Document not found' } - } - - return { - hasAccess: true, - document: doc, - knowledgeBase: kbAccess.knowledgeBase!, - } -} - -/** - * Check if a user has read access to a document within a knowledge base. - */ -export async function checkDocumentAccess( - knowledgeBaseId: string, - documentId: string, - userId: string -): Promise { - return resolveDocumentAccess(knowledgeBaseId, documentId, userId, false) -} - -/** - * Check if a user has write access to a specific document. - * Write access is granted if user has write access to the knowledge base. - */ -export async function checkDocumentWriteAccess( - knowledgeBaseId: string, - documentId: string, - userId: string -): Promise { - return resolveDocumentAccess(knowledgeBaseId, documentId, userId, true) -} - -/** - * Resolve chunk access within a document/knowledge base, gated by read or write - * permission on the KB. The document must exist and be fully processed - * (`processingStatus === 'completed'`) before its chunks are accessible. - */ -async function resolveChunkAccess( - knowledgeBaseId: string, - documentId: string, - chunkId: string, - userId: string, - requireWrite: boolean -): Promise { - const kbAccess = await resolveKnowledgeBaseAccess(knowledgeBaseId, userId, requireWrite) - - if (!kbAccess.hasAccess) { - return { - hasAccess: false, - notFound: kbAccess.notFound, - reason: kbAccess.notFound ? 'Knowledge base not found' : 'Unauthorized knowledge base access', - } - } - - const doc = await getKnowledgeDocument(knowledgeBaseId, documentId) - if (!doc) { - return { hasAccess: false, notFound: true, reason: 'Document not found' } - } - - const docData = doc - - // Chunks are only accessible once the document has finished processing. - if (docData.processingStatus !== 'completed') { - return { - hasAccess: false, - reason: `Document is not ready for access (status: ${docData.processingStatus})`, - } - } - - const chunk = await db - .select() - .from(embedding) - .where(and(eq(embedding.id, chunkId), eq(embedding.documentId, documentId))) - .limit(1) - - if (chunk.length === 0) { - return { hasAccess: false, notFound: true, reason: 'Chunk not found' } - } - - return { - hasAccess: true, - chunk: chunk[0] as EmbeddingData, - document: docData, - knowledgeBase: kbAccess.knowledgeBase!, - } -} - -/** - * Check if a user has read access to a chunk within a document and knowledge base. - */ -export async function checkChunkAccess( - knowledgeBaseId: string, - documentId: string, - chunkId: string, - userId: string -): Promise { - return resolveChunkAccess(knowledgeBaseId, documentId, chunkId, userId, false) -} - -/** - * Check if a user has write access to a chunk. - * - * Mirrors {@link checkChunkAccess} but requires write/admin on the knowledge - * base's workspace (or KB ownership for legacy KBs), matching the permission - * needed to create chunks. Used for chunk mutation (update and delete) so those - * operations require the same permission as creation rather than read. - */ -export async function checkChunkWriteAccess( - knowledgeBaseId: string, - documentId: string, - chunkId: string, - userId: string -): Promise { - return resolveChunkAccess(knowledgeBaseId, documentId, chunkId, userId, true) -} diff --git a/apps/sim/app/api/v1/knowledge/[id]/documents/[documentId]/route.ts b/apps/sim/app/api/v1/knowledge/[id]/documents/[documentId]/route.ts index b7a363fc11c..7ad5558439e 100644 --- a/apps/sim/app/api/v1/knowledge/[id]/documents/[documentId]/route.ts +++ b/apps/sim/app/api/v1/knowledge/[id]/documents/[documentId]/route.ts @@ -1,6 +1,3 @@ -import { db } from '@sim/db' -import { document, knowledgeConnector } from '@sim/db/schema' -import { and, eq, isNull } from 'drizzle-orm' import { type NextRequest, NextResponse } from 'next/server' import { v1DeleteKnowledgeDocumentContract, @@ -12,8 +9,14 @@ import { statusForOrchestrationError, } from '@/lib/core/orchestration/types' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { getKnowledgeDocument } from '@/lib/knowledge/documents/service' import { performDeleteKnowledgeDocument } from '@/lib/knowledge/orchestration' -import { handleError, resolveKnowledgeBase, serializeDate } from '@/app/api/v1/knowledge/utils' +import { + handleError, + resolveKnowledgeBase, + resolveV1KnowledgeAccessScope, + serializeDate, +} from '@/app/api/v1/knowledge/utils' import { authenticateRequest, v1ValidationErrorResponse } from '@/app/api/v1/middleware' export const dynamic = 'force-dynamic' @@ -46,45 +49,16 @@ export const GET = withRouteHandler( ) if (result instanceof NextResponse) return result - const docs = await db - .select({ - id: document.id, - knowledgeBaseId: document.knowledgeBaseId, - filename: document.filename, - fileSize: document.fileSize, - mimeType: document.mimeType, - processingStatus: document.processingStatus, - processingError: document.processingError, - processingStartedAt: document.processingStartedAt, - processingCompletedAt: document.processingCompletedAt, - chunkCount: document.chunkCount, - tokenCount: document.tokenCount, - characterCount: document.characterCount, - enabled: document.enabled, - uploadedAt: document.uploadedAt, - connectorId: document.connectorId, - connectorType: knowledgeConnector.connectorType, - sourceUrl: document.sourceUrl, - }) - .from(document) - .leftJoin(knowledgeConnector, eq(document.connectorId, knowledgeConnector.id)) - .where( - and( - eq(document.id, documentId), - eq(document.knowledgeBaseId, knowledgeBaseId), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt) - ) - ) - .limit(1) + const doc = await getKnowledgeDocument( + knowledgeBaseId, + documentId, + await resolveV1KnowledgeAccessScope(userId, rateLimit, parsed.data.query.workspaceId) + ) - if (docs.length === 0) { + if (!doc) { return NextResponse.json({ error: 'Document not found' }, { status: 404 }) } - const doc = docs[0] - return NextResponse.json({ success: true, data: { @@ -139,21 +113,13 @@ export const DELETE = withRouteHandler( ) if (result instanceof NextResponse) return result - const docs = await db - .select({ id: document.id, filename: document.filename }) - .from(document) - .where( - and( - eq(document.id, documentId), - eq(document.knowledgeBaseId, knowledgeBaseId), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt) - ) - ) - .limit(1) + const doc = await getKnowledgeDocument( + knowledgeBaseId, + documentId, + await resolveV1KnowledgeAccessScope(userId, rateLimit, parsed.data.query.workspaceId) + ) - if (docs.length === 0) { + if (!doc) { return NextResponse.json({ error: 'Document not found' }, { status: 404 }) } @@ -163,7 +129,7 @@ export const DELETE = withRouteHandler( name: result.kb.name, workspaceId: parsed.data.query.workspaceId, }, - document: { id: documentId, filename: docs[0].filename }, + document: { id: documentId, filename: doc.filename }, userId, source: 'api', requestId, diff --git a/apps/sim/app/api/v1/knowledge/[id]/documents/route.ts b/apps/sim/app/api/v1/knowledge/[id]/documents/route.ts index e04b3370cbc..ba403a072e8 100644 --- a/apps/sim/app/api/v1/knowledge/[id]/documents/route.ts +++ b/apps/sim/app/api/v1/knowledge/[id]/documents/route.ts @@ -26,7 +26,12 @@ import { performUploadKnowledgeDocument } from '@/lib/knowledge/orchestration' import { uploadWorkspaceFile } from '@/lib/uploads/contexts/workspace' import { EXACT_EMPTY_WORKSPACE_FILE_SECRET_PROVENANCE } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' import { validateFileType } from '@/lib/uploads/utils/validation' -import { handleError, resolveKnowledgeBase, serializeDate } from '@/app/api/v1/knowledge/utils' +import { + handleError, + resolveKnowledgeBase, + resolveV1KnowledgeAccessScope, + serializeDate, +} from '@/app/api/v1/knowledge/utils' import { authenticateRequest, v1ValidationErrorResponse } from '@/app/api/v1/middleware' export const dynamic = 'force-dynamic' @@ -73,7 +78,8 @@ export const GET = withRouteHandler(async (request: NextRequest, context: Docume sortBy: sortBy as DocumentSortField, sortOrder: sortOrder as SortOrder, }, - requestId + requestId, + await resolveV1KnowledgeAccessScope(userId, rateLimit, workspaceId) ) return NextResponse.json({ diff --git a/apps/sim/app/api/v1/knowledge/search/route.test.ts b/apps/sim/app/api/v1/knowledge/search/route.test.ts index 7c5cbc2fc23..40747a907a4 100644 --- a/apps/sim/app/api/v1/knowledge/search/route.test.ts +++ b/apps/sim/app/api/v1/knowledge/search/route.test.ts @@ -44,9 +44,13 @@ const SYSTEM_BILLING_ATTRIBUTION = { payerSubscription: null, } +/** The route's defaults depend on member-access availability; pin it so the local flag cannot change the expectations. */ +vi.mock('@/lib/knowledge/access/availability', () => ({ + isKnowledgeMemberAccessAvailable: async () => false, +})) + vi.mock('@/lib/knowledge/search/queries', () => ({ executeKnowledgeSearch: mockExecuteKnowledgeSearch, - generateSearchEmbedding: mockGenerateSearchEmbedding, getDocumentMetadataByIds: mockGetDocumentMetadataByIds, })) @@ -63,17 +67,23 @@ vi.mock('@/lib/billing/core/billing-attribution', () => ({ })) vi.mock('@/lib/knowledge/embeddings', () => ({ + generateSearchEmbedding: mockGenerateSearchEmbedding, recordSearchEmbeddingUsage: mockRecordSearchEmbeddingUsage, })) vi.mock('@/app/api/v1/middleware', () => ({ authenticateRequest: mockAuthenticateRequest, validateWorkspaceAccess: mockValidateWorkspaceAccess, + capabilityGovernedUserId: (rateLimit: { keyType?: string; userId?: string }) => + rateLimit.keyType === 'workspace' ? null : (rateLimit.userId ?? null), v1ValidationErrorResponse: (e: { issues: unknown[] }) => NextResponse.json({ error: 'Validation error', details: e.issues }, { status: 400 }), })) vi.mock('@/app/api/v1/knowledge/utils', () => ({ + resolveV1KnowledgeAccessScope: vi + .fn() + .mockResolvedValue({ kind: 'workspace', tokens: ['pub', 'ws'] }), handleError: (e: unknown) => new Response(JSON.stringify({ error: getErrorMessage(e, 'error') }), { status: 500, diff --git a/apps/sim/app/api/v1/knowledge/search/route.ts b/apps/sim/app/api/v1/knowledge/search/route.ts index 7f331dbe0a4..05037fb52f0 100644 --- a/apps/sim/app/api/v1/knowledge/search/route.ts +++ b/apps/sim/app/api/v1/knowledge/search/route.ts @@ -8,10 +8,10 @@ import { } from '@/lib/billing/core/billing-attribution' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { ALL_TAG_SLOTS } from '@/lib/knowledge/constants' -import { recordSearchEmbeddingUsage } from '@/lib/knowledge/embeddings' +import { generateSearchEmbedding, recordSearchEmbeddingUsage } from '@/lib/knowledge/embeddings' +import { resolveKnowledgeSearchDefaults } from '@/lib/knowledge/search/defaults' import { executeKnowledgeSearch, - generateSearchEmbedding, getDocumentMetadataByIds, type SearchResult, } from '@/lib/knowledge/search/queries' @@ -19,9 +19,10 @@ import { getDocumentTagDefinitions } from '@/lib/knowledge/tags/service' import { buildUndefinedTagsError, validateTagValue } from '@/lib/knowledge/tags/utils' import type { StructuredFilter } from '@/lib/knowledge/types' import { checkKnowledgeBaseAccess, type KnowledgeBaseAccessResult } from '@/app/api/knowledge/utils' -import { handleError } from '@/app/api/v1/knowledge/utils' +import { handleError, resolveV1KnowledgeAccessScope } from '@/app/api/v1/knowledge/utils' import { authenticateRequest, + capabilityGovernedUserId, v1ValidationErrorResponse, validateWorkspaceAccess, } from '@/app/api/v1/middleware' @@ -46,7 +47,13 @@ export const POST = withRouteHandler(async (request: NextRequest) => { ) if (!parsed.success) return parsed.response - const { workspaceId, topK, query, tagFilters, searchMode } = parsed.data.body + const { + workspaceId, + topK, + query, + tagFilters, + searchMode: requestedSearchMode, + } = parsed.data.body const accessError = await validateWorkspaceAccess( rateLimit, @@ -190,12 +197,23 @@ export const POST = withRouteHandler(async (request: NextRequest) => { let results: SearchResult[] let queryEmbeddingIsBYOK: boolean | null = null + const [access, { searchMode, boostRecency }] = await Promise.all([ + resolveV1KnowledgeAccessScope(userId, rateLimit, workspaceId), + resolveKnowledgeSearchDefaults({ + workspaceId, + /** A personal key acts as its user; a workspace key has no person behind it. */ + userId: capabilityGovernedUserId(rateLimit) ?? undefined, + requestedMode: requestedSearchMode, + }), + ]) if (!hasQuery && hasFilters) { results = await executeKnowledgeSearch({ knowledgeBaseIds: accessibleKbIds, topK, + access, searchMode, + boostRecency, structuredFilters, }) } else if (hasQuery) { @@ -209,7 +227,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { results = await executeKnowledgeSearch({ knowledgeBaseIds: accessibleKbIds, topK, + access, searchMode, + boostRecency, query, queryVector, structuredFilters: hasFilters ? structuredFilters : undefined, @@ -253,7 +273,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => { }) const documentIds = results.map((r) => r.documentId) - const documentMetadataMap = await getDocumentMetadataByIds(documentIds) + const documentMetadataMap = await getDocumentMetadataByIds(documentIds, access) return NextResponse.json({ success: true, diff --git a/apps/sim/app/api/v1/knowledge/utils.ts b/apps/sim/app/api/v1/knowledge/utils.ts index 9995f7a3144..95655cbe756 100644 --- a/apps/sim/app/api/v1/knowledge/utils.ts +++ b/apps/sim/app/api/v1/knowledge/utils.ts @@ -1,6 +1,11 @@ import { createLogger } from '@sim/logger' import { NextResponse } from 'next/server' import { validationErrorResponseFromError } from '@/lib/api/server' +import { + resolveUserKnowledgeAccessScope, + WORKSPACE_ACCESS_SCOPE, +} from '@/lib/knowledge/access/scope' +import type { KnowledgeAccessScope } from '@/lib/knowledge/access/types' import { getKnowledgeBaseById } from '@/lib/knowledge/service' import type { KnowledgeBaseWithCounts } from '@/lib/knowledge/types' import { @@ -47,6 +52,19 @@ export async function resolveKnowledgeBase( return { kb } } +/** + * The document-access scope of a v1 API caller. A personal key acts as its + * user; a workspace key has no person behind it and reads as the workspace. + */ +export async function resolveV1KnowledgeAccessScope( + userId: string, + rateLimit: { keyType?: 'personal' | 'workspace' }, + workspaceId: string | undefined +): Promise { + if (rateLimit.keyType === 'workspace') return WORKSPACE_ACCESS_SCOPE + return resolveUserKnowledgeAccessScope(userId, workspaceId) +} + /** * Serializes a date value for JSON responses. */ diff --git a/apps/sim/app/api/workflows/[id]/execute/route.ts b/apps/sim/app/api/workflows/[id]/execute/route.ts index 53e07eb7e1d..2d99b7a7a7f 100644 --- a/apps/sim/app/api/workflows/[id]/execute/route.ts +++ b/apps/sim/app/api/workflows/[id]/execute/route.ts @@ -1497,6 +1497,7 @@ async function handleExecutePost( fileKeys: outputFileKeys, allowLargeValueWorkflowScope, userId: actorUserId, + principal: executionPrincipal, maxBytes: base64MaxBytes, preserveLargeValueMetadata: true, })) as NormalizedBlockOutput) @@ -1701,6 +1702,7 @@ async function handleExecutePost( workspaceId, workflowId, userId: actorUserId, + principal: executionPrincipal, allowLargeValueWorkflowScope, requestSignal: req.signal, requestHeaders: req.headers, @@ -2303,6 +2305,7 @@ async function handleExecutePost( fileKeys: outputFileKeys, allowLargeValueWorkflowScope, userId: actorUserId, + principal: executionPrincipal, maxBytes: base64MaxBytes, preserveLargeValueMetadata: true, }) diff --git a/apps/sim/app/workspace/[workspaceId]/components/integration-tabs-header/integration-tabs-header.tsx b/apps/sim/app/workspace/[workspaceId]/components/integration-tabs-header/integration-tabs-header.tsx index 53efc5a0175..60863ec586f 100644 --- a/apps/sim/app/workspace/[workspaceId]/components/integration-tabs-header/integration-tabs-header.tsx +++ b/apps/sim/app/workspace/[workspaceId]/components/integration-tabs-header/integration-tabs-header.tsx @@ -3,19 +3,19 @@ import { ChipLink, cn } from '@sim/emcn' import { HEADER_ACTION_CLUSTER, PAGE_HEADER_BAR } from '@/components/page-header-bar' interface IntegrationTabsHeaderProps { - active: 'integrations' | 'skills' + active: 'integrations' | 'skills' | 'search' workspaceId: string /** Trailing actions for the owning page (e.g. skills' "Add skill"). */ rightSlot?: ReactNode } /** - * Top-of-page tab header shared by the Integrations and Skills pages — two halves - * of one surface, so each highlights itself and links to its sibling. + * Top-of-page tab header shared by the Integrations, Skills, and Search pages — + * three views of one surface, so each highlights itself and links to its siblings. * * Lives in the shared workspace components rather than under `integrations/` - * because both pages own it equally; its former home made Skills reach across into - * a sibling feature for its own chrome. + * because every page owns it equally; its former home made Skills reach across + * into a sibling feature for its own chrome. * * The `gap-1` is explicit because chips carry no outer margin — the parent owns the * space between them. @@ -33,6 +33,9 @@ export function IntegrationTabsHeader({ Skills + + Search + {rightSlot &&
{rightSlot}
} ) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/index.ts index 8425c127a6e..fb354716ea3 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/index.ts @@ -12,5 +12,4 @@ export { useMothershipResources, } from './mothership-resources-context' export { QueuedMessages } from './queued-messages' -export { SuggestedActions } from './suggested-actions' export { UserInput, type UserInputHandle } from './user-input' diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/index.ts new file mode 100644 index 00000000000..a170379c468 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/index.ts @@ -0,0 +1,5 @@ +export { + groupResultsByDocument, + indexingSourceNames, + KnowledgeSearchResults, +} from './knowledge-search-results' diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx new file mode 100644 index 00000000000..0be852d394d --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.test.tsx @@ -0,0 +1,67 @@ +/** + * @vitest-environment jsdom + */ +import { describe, expect, it, vi } from 'vitest' +import type { WorkspaceMemberConnector } from '@/hooks/queries/kb/connectors' + +vi.mock('@/hooks/queries/kb/connectors', () => ({ useWorkspaceMemberConnectors: vi.fn() })) +vi.mock('@/hooks/queries/kb/knowledge', () => ({ + useKnowledgeBasesQuery: vi.fn(), + useWorkspaceKnowledgeSearch: vi.fn(), +})) +vi.mock('@/app/workspace/[workspaceId]/providers/workspace-host-provider', () => ({ + useWorkspaceHostContext: vi.fn(), +})) +vi.mock( + '@/app/workspace/[workspaceId]/home/components/message-content/components/source-card', + () => ({ SourceCard: () => null }) +) + +import { indexingSourceNames } from '@/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results' + +function memberConnector( + overrides: Partial = {} +): WorkspaceMemberConnector { + return { + knowledgeBaseId: 'kb-1', + knowledgeBaseName: 'Sim Search', + connectorId: 'connector-1', + connectorType: 'google_drive', + memberSyncStatus: 'running', + viewerMembership: 'connected', + viewerDocumentCount: 0, + ...overrides, + } +} + +describe('indexingSourceNames', () => { + it('names each source still indexing for the viewer once, in the searched bases only', () => { + const names = indexingSourceNames( + [ + memberConnector({ connectorId: 'a', connectorType: 'google_drive' }), + memberConnector({ + connectorId: 'b', + connectorType: 'google_drive', + knowledgeBaseId: 'kb-2', + }), + memberConnector({ connectorId: 'c', connectorType: 'slack', memberSyncStatus: 'pending' }), + memberConnector({ connectorId: 'd', connectorType: 'notion', knowledgeBaseId: 'kb-3' }), + ], + ['kb-1', 'kb-2'] + ) + + expect(names).toEqual(['Google Drive', 'Slack']) + }) + + it('ignores sources that are idle or not connected for the viewer', () => { + expect( + indexingSourceNames( + [ + memberConnector({ connectorId: 'a', memberSyncStatus: 'idle' }), + memberConnector({ connectorId: 'b', viewerMembership: 'invited' }), + ], + ['kb-1'] + ) + ).toEqual([]) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx new file mode 100644 index 00000000000..5c9f0a1b469 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/knowledge-search-results/knowledge-search-results.tsx @@ -0,0 +1,321 @@ +'use client' + +import { useMemo } from 'react' +import { Button, Chip, OverflowText } from '@sim/emcn' +import { FileText } from '@sim/emcn/icons' +import { formatDate } from '@sim/utils/formatting' +import { useQueryStates } from 'nuqs' +import type { WorkspaceKnowledgeSearchResult } from '@/lib/api/contracts/knowledge' +import { matchSnippet } from '@/lib/knowledge/search/snippet' +import { connectorDisplayName } from '@/lib/sim-search/connectors' +import { searchedKnowledgeBases } from '@/lib/sim-search/knowledge-bases' +import { + highlightTerms, + SOURCE_ROW_CLASSES, + SOURCE_ROW_MARK_CLASSES, + SourceCard, +} from '@/app/workspace/[workspaceId]/home/components/message-content/components/source-card' +import { + isHttpUrl, + type SourceTagData, +} from '@/app/workspace/[workspaceId]/home/components/message-content/components/special-tags' +import { isIndexing } from '@/app/workspace/[workspaceId]/home/components/search-sources' +import { + resourceUrlKeys, + searchFilterParsers, + UPDATED_WINDOWS, +} from '@/app/workspace/[workspaceId]/home/search-params' +import { useWorkspaceHostContext } from '@/app/workspace/[workspaceId]/providers/workspace-host-provider' +import { + useWorkspaceMemberConnectors, + type WorkspaceMemberConnector, +} from '@/hooks/queries/kb/connectors' +import { useKnowledgeBasesQuery, useWorkspaceKnowledgeSearch } from '@/hooks/queries/kb/knowledge' + +const EMPTY_MEMBER_CONNECTORS: WorkspaceMemberConnector[] = [] + +/** Filters appear only once a list is long and mixed enough for them to help. */ +const FILTERS_MIN_RESULTS = 10 +const DAY_MS = 24 * 60 * 60 * 1000 +/** Every result without a connector is an upload; the filter names them so. */ +const UPLOAD_SOURCE = 'upload' + +/** + * One card per document, keeping the best-ranked chunk of each: the list is + * already in rank order, so the first chunk seen for a document is its best. + */ +export function groupResultsByDocument( + results: readonly WorkspaceKnowledgeSearchResult[] +): WorkspaceKnowledgeSearchResult[] { + const seen = new Set() + const grouped: WorkspaceKnowledgeSearchResult[] = [] + for (const result of results) { + if (seen.has(result.documentId)) continue + seen.add(result.documentId) + grouped.push(result) + } + return grouped +} + +/** + * The names of the sources still indexing for the viewer among the bases the + * search spans, each once. A base outside the search cannot grow its results, + * so its indexing is not the reader's concern here. + */ +export function indexingSourceNames( + memberConnectors: readonly WorkspaceMemberConnector[], + knowledgeBaseIds: readonly string[] +): string[] { + const searched = new Set(knowledgeBaseIds) + return [ + ...new Set( + memberConnectors + .filter((connection) => searched.has(connection.knowledgeBaseId) && isIndexing(connection)) + .map((connection) => connectorDisplayName(connection.connectorType)) + ), + ] +} + +/** + * A result as the source card renders it: the row's second line names the + * source app, or the knowledge base for an upload. A document without an + * http(s) source URL cannot be opened, and a connector-supplied value of any + * other scheme is never handed to the browser as a link. + */ +function toSource(result: WorkspaceKnowledgeSearchResult, query: string): SourceTagData | null { + if (!isHttpUrl(result.sourceUrl)) return null + return { + url: result.sourceUrl, + title: result.documentName ?? undefined, + siteName: result.connectorType + ? connectorDisplayName(result.connectorType) + : result.knowledgeBaseName || undefined, + connectorType: result.connectorType ?? undefined, + snippet: matchSnippet(result.content, query), + author: result.author ?? undefined, + updatedAt: result.sourceModifiedAt ?? undefined, + } +} + +/** + * Arrow keys walk the result links, the way a search page does; Enter on a + * focused link opens it natively. Focus stops at either end. + */ +function handleResultsKeyDown(event: React.KeyboardEvent) { + if (event.key !== 'ArrowDown' && event.key !== 'ArrowUp') return + const links = [...event.currentTarget.querySelectorAll('a[data-source-link]')] + if (links.length === 0) return + const index = links.findIndex((link) => link === document.activeElement) + const next = + event.key === 'ArrowDown' ? Math.min(index + 1, links.length - 1) : Math.max(index - 1, 0) + if (next === index) return + event.preventDefault() + links[next].focus() +} + +interface UnlinkedResultRowProps { + result: WorkspaceKnowledgeSearchResult + query: string +} + +/** + * A document with nowhere to open, such as an upload: the same row as a + * linked result, with the file mark in place of a brand mark, so the list's + * columns and the matched passage stay aligned whatever the document is. + */ +function UnlinkedResultRow({ result, query }: UnlinkedResultRowProps) { + const meta = [ + result.knowledgeBaseName, + result.author, + result.sourceModifiedAt ? formatDate(new Date(result.sourceModifiedAt)) : null, + ].filter((part): part is string => Boolean(part)) + return ( +
+ + + +
+ + +

+ {highlightTerms(matchSnippet(result.content, query), query)} +

+
+
+ ) +} + +interface KnowledgeSearchResultsProps { + workspaceId: string + query: string + /** Asks the agent about one document; the prompt names it and links to it. */ + onSummarize: (prompt: string) => void + /** Asks the agent the query itself, for a prose answer with citations. */ + onAnswer: (query: string) => void +} + +/** + * The composer's Search mode: the documents the signed-in person may read that + * match their query, across every knowledge base in the workspace, as rows + * that open the source. A header says how many and that the search ran as + * them; while a connected source is still indexing it says so, and the list + * grows as documents land. Filters by source and recency appear only once the + * list is long and mixed enough to need them, and live in the URL beside the + * query so a filtered search is a shareable link. + */ +export function KnowledgeSearchResults({ + workspaceId, + query, + onSummarize, + onAnswer, +}: KnowledgeSearchResultsProps) { + const { + data: knowledgeBases = [], + isPending: basesPending, + error: basesError, + } = useKnowledgeBasesQuery(workspaceId) + const knowledgeBaseIds = searchedKnowledgeBases(knowledgeBases, workspaceId).map((kb) => kb.id) + const { + data: results, + isPending, + isFetching, + isPlaceholderData, + error, + } = useWorkspaceKnowledgeSearch(workspaceId, knowledgeBaseIds, query) + const { features } = useWorkspaceHostContext() + /** + * Judged by the workspace, as the server judges it: with per-member access + * off, member-scoped documents are hidden, so no source is indexing anything + * the viewer will see, and the list is not worth asking for. + */ + const memberAccessAvailable = features?.knowledgeMemberAccess === true + const { data: memberConnectorRows } = useWorkspaceMemberConnectors(workspaceId, { + enabled: memberAccessAvailable, + }) + /** Rows cached before the feature went off are not this surface's to show. */ + const memberConnectors = memberAccessAvailable + ? (memberConnectorRows ?? EMPTY_MEMBER_CONNECTORS) + : EMPTY_MEMBER_CONNECTORS + const indexing = indexingSourceNames(memberConnectors, knowledgeBaseIds) + const documents = useMemo(() => groupResultsByDocument(results ?? []), [results]) + const sourceTypes = useMemo( + () => [...new Set(documents.map((result) => result.connectorType ?? UPLOAD_SOURCE))], + [documents] + ) + const [filters, setFilters] = useQueryStates(searchFilterParsers, resourceUrlKeys) + const filtersActive = filters.source !== null || filters.updated !== 'any' + /** The controls appear once the list is long and mixed, and stay while a filter from the link is active. */ + const showFilters = + filtersActive || (documents.length >= FILTERS_MIN_RESULTS && sourceTypes.length > 1) + const visible = useMemo(() => { + if (!filtersActive) return documents + const window = UPDATED_WINDOWS.find((entry) => entry.id === filters.updated) + const cutoff = window?.days ? Date.now() - window.days * DAY_MS : null + return documents.filter((result) => { + if (filters.source && (result.connectorType ?? UPLOAD_SOURCE) !== filters.source) return false + if (cutoff !== null) { + const modified = result.sourceModifiedAt ? Date.parse(result.sourceModifiedAt) : Number.NaN + if (Number.isNaN(modified) || modified < cutoff) return false + } + return true + }) + }, [documents, filtersActive, filters.source, filters.updated]) + + const failure = basesError ?? error + if (failure) { + return

{failure.message}

+ } + if (!basesPending && knowledgeBaseIds.length === 0) { + return ( +

+ Nothing to search yet. Clear the query and connect a source to index what you can open. +

+ ) + } + /** Kept results belong to the previous query; a new query shows its own state. */ + if (isPending || isPlaceholderData || (isFetching && !results)) { + return

Searching…

+ } + + const indexingNote = + indexing.length > 0 + ? `Still indexing ${indexing.join(', ')}; results grow as documents land.` + : null + + return ( +
+
+ + + {documents.length === 1 ? '1 document' : `${documents.length} documents`} + + {' · searched as you'} + {indexingNote && {indexingNote}} + + +
+ {showFilters && ( +
+ setFilters({ source: null })} + > + All sources + + {sourceTypes.map((type) => ( + setFilters({ source: filters.source === type ? null : type })} + > + {type === UPLOAD_SOURCE ? 'Uploads' : connectorDisplayName(type)} + + ))} + + {UPDATED_WINDOWS.map((window) => ( + setFilters({ updated: window.id })} + > + {window.label} + + ))} +
+ )} + {visible.length === 0 ? ( +

+ {documents.length === 0 + ? `No documents you can read match “${query}”.` + : 'No documents match these filters.'} +

+ ) : ( +
+ {visible.map((result) => { + const source = toSource(result, query) + return source ? ( + + onSummarize(`Summarize "${cited.title ?? cited.url}" (${cited.url})`) + } + /> + ) : ( + + ) + })} +
+ )} +
+ ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.test.ts index 1b3e5d105d9..eb9f86f6035 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.test.ts @@ -15,6 +15,14 @@ describe('sanitizeChatDisplayContent', () => { ) }) + it('unwraps source tags from inline code spans', () => { + const content = '`Block them first. {"url":"https://docs.github.com/a"}`' + + expect(sanitizeChatDisplayContent(content)).toBe( + 'Block them first. {"url":"https://docs.github.com/a"}' + ) + }) + it('removes hidden internal references wrapped in inline code', () => { const content = 'Read `internal/tool-results/read-1.md` and found the issue.' diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.tsx index c143c72bb4a..d75aedac99f 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-content.tsx @@ -1,6 +1,15 @@ 'use client' -import { type ComponentPropsWithoutRef, memo, useEffect, useMemo, useRef, useState } from 'react' +import { + type ComponentPropsWithoutRef, + createContext, + memo, + useContext, + useEffect, + useMemo, + useRef, + useState, +} from 'react' import { Streamdown } from 'streamdown' import 'streamdown/styles.css' // prismjs core must load before its language components — they register on the @@ -15,10 +24,15 @@ import { Checkbox, CopyCodeButton, cn, languages, highlight as prismHighlight } import { decodeVfsSegmentSafe } from '@/lib/copilot/vfs/path-utils' import { extractTextContent } from '@/lib/core/utils/react-node-text' import { ContextMentionIcon } from '@/app/workspace/[workspaceId]/home/components/context-mention-icon' +import { + SourceChip, + sourceLabel, +} from '@/app/workspace/[workspaceId]/home/components/message-content/components/source-chip' import { type ContentSegment, type CredentialSubmissionPayload, parseSpecialTags, + type SourceTagData, SpecialTags, } from '@/app/workspace/[workspaceId]/home/components/message-content/components/special-tags' import type { @@ -108,9 +122,47 @@ function nextInlineSegmentLabel(segment?: ContentSegment): string { // Thinking segments are never rendered, so they contribute no following text. if (segment.type === 'text') return segment.content if (segment.type === 'workspace_resource') return segment.data.title || segment.data.id || '' + if (segment.type === 'source') return sourceLabel(segment.data) return '' } +/** + * The `` payloads of the segment being rendered, in emission order. An + * inline citation is written into the markdown as a link to a sentinel + * fragment carrying the payload's index, so it flows with its paragraph, and + * the link renderer resolves the index back through this context — the + * component map is static, so it is the one channel from segment data into it. + */ +const SourceRefsContext = createContext([]) + +/** + * Fragment prefix of a generated citation link. Internal — never navigated — + * and deliberately not a name the model would write on its own; an index that + * resolves to no parsed source falls back to the link text. + */ +const SOURCE_LINK_PREFIX = '#sim-source-ref-' + +interface SourceReferenceProps { + index: number + children?: React.ReactNode +} + +/** The inline citation chip; a dangling index falls back to the link text. */ +function SourceReference({ index, children }: SourceReferenceProps) { + const source = useContext(SourceRefsContext)[index] + if (!source) return <>{children} + return +} + +/** + * A source's name as a Markdown link label. A site name or knowledge-base + * name is free text: an unescaped `]` would end the label early and a `*` or + * `_` would style it, so the delimiters are backslash-escaped. + */ +function escapeLinkLabel(label: string): string { + return label.replace(/[\\[\]*_`<>]/g, '\\$&') +} + function appendInlineReferenceMarkdown( currentMarkdown: string, referenceMarkdown: string, @@ -263,6 +315,13 @@ const MARKDOWN_COMPONENTS = { ) }, a({ children, href }: { children?: React.ReactNode; href?: string }) { + if (href?.startsWith(SOURCE_LINK_PREFIX)) { + return ( + + {children} + + ) + } if (href?.startsWith('#wsres-')) { const match = href.match(/^#wsres-(\w+)-(.+)$/) const type = match?.[1] @@ -566,14 +625,20 @@ function ChatContentInner({ type BlockSegment = Exclude< ContentSegment, - { type: 'text' } | { type: 'thinking' } | { type: 'workspace_resource' } + { type: 'text' } | { type: 'thinking' } | { type: 'workspace_resource' } | { type: 'source' } > type RenderGroup = | { kind: 'inline'; markdown: string } | { kind: 'block'; segment: BlockSegment; index: number } + const sourceRefs = useMemo( + () => parsed.segments.flatMap((segment) => (segment.type === 'source' ? [segment.data] : [])), + [parsed] + ) + const groups: RenderGroup[] = [] let pendingMarkdown = '' + let sourceIndex = 0 const flushMarkdown = () => { if (pendingMarkdown.trim()) { @@ -596,6 +661,16 @@ function ChatContentInner({ `[${label}](<#wsres-${s.data.type}-${ref}>)`, nextSegment ) + } else if (s.type === 'source') { + // A citation always stands off from the sentence it supports, even when + // the model closes the sentence on punctuation the word-boundary rule + // would otherwise glue the chip to. + if (pendingMarkdown && !/\s$/.test(pendingMarkdown)) pendingMarkdown += ' ' + pendingMarkdown = appendInlineReferenceMarkdown( + pendingMarkdown, + `[${escapeLinkLabel(sourceLabel(s.data))}](<${SOURCE_LINK_PREFIX}${sourceIndex++}>)`, + nextSegment + ) } else if (s.type === 'thinking') { // Model-emitted tag bodies are reasoning, not answer text — // never rendered (matches the block-level thinking omission in @@ -621,40 +696,42 @@ function ChatContentInner({ * the new special block mounts. */ return ( -
- {groups.map((group, i) => { - if (group.kind === 'inline') { - return ( -
:first-child]:mt-0 [&>:last-child]:mb-0')} - > - +
+ {groups.map((group, i) => { + if (group.kind === 'inline') { + return ( +
:first-child]:mt-0 [&>:last-child]:mb-0')} > - {group.markdown} - -
+ + {group.markdown} + +
+ ) + } + return ( + ) - } - return ( - - ) - })} -
+ })} +
+ ) } diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-sanitize.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-sanitize.ts index 4e0792df0ab..1d210bd1b92 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-sanitize.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/chat-sanitize.ts @@ -2,7 +2,9 @@ const HIDDEN_INLINE_REFERENCE_PATTERN = /`[^`\n]*(?:internal\/tool-results\/|internal\/blocktips\/|components\/integrations\/[^`\n]*README)[^`\n]*`/g /** - * A complete workspace-resource tag: opener, payload, closer. + * A complete inline-chip tag — `` or `` — as + * opener, payload, closer. Both are JSON-bodied tags the model places inside a + * sentence, so both attract the same stray backticks. * * Two constraints on the payload, both load-bearing: * @@ -19,10 +21,10 @@ const HIDDEN_INLINE_REFERENCE_PATTERN = * is rare; the failure it replaces corrupts a whole message and is common. */ const COMPLETE_TAG_SOURCE = - '(?:(?!)[^`])*?<\\/workspace_resource>' + '<(?workspace_resource|source)>(?:(?!<\\k>)[^`])*?<\\/\\k>' /** Non-global so {@link RegExp.test} has no `lastIndex` to carry between calls. */ -const COMPLETE_WORKSPACE_RESOURCE_TAG = new RegExp(COMPLETE_TAG_SOURCE) +const COMPLETE_INLINE_CHIP_TAG = new RegExp(COMPLETE_TAG_SOURCE) /** * One left-to-right pass over the two things that can own a backtick: an inline @@ -57,7 +59,7 @@ export function sanitizeChatDisplayContent(content: string): string { // lifts the tag out either way, so leaving the delimiters would strand a // pair of backticks around a hole. Anything else is someone else's span. const inner = match.slice(1, -1) - return COMPLETE_WORKSPACE_RESOURCE_TAG.test(inner) ? inner : match + return COMPLETE_INLINE_CHIP_TAG.test(inner) ? inner : match }) .replace(HIDDEN_INLINE_REFERENCE_PATTERN, '') } diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/external-link.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/external-link.tsx index 405acc7a9d4..fd9661d2151 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/external-link.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/external-link.tsx @@ -6,7 +6,7 @@ import { faviconUrl } from '@/lib/core/utils/favicon' import { useLinkPreview } from '@/hooks/queries/link-preview' /** Hides a favicon img that failed to load so the link degrades to plain text. */ -function hideBrokenFavicon(e: React.SyntheticEvent): void { +export function hideBrokenFavicon(e: React.SyntheticEvent): void { e.currentTarget.style.display = 'none' } @@ -44,7 +44,10 @@ interface ExternalLinkProps { * which the shell routes to the system browser. In a web browser this is a * no-op and the link opens a new tab as usual. */ -function handleLinkClick(event: React.MouseEvent, href: string): void { +export function handleExternalLinkClick( + event: React.MouseEvent, + href: string +): void { if (event.metaKey || event.ctrlKey || event.shiftKey || event.button !== 0) return if (!shouldOpenInBrowserPanel(href)) return event.preventDefault() @@ -63,7 +66,7 @@ export function ExternalLink({ href, hostname, children }: ExternalLinkProps) { className='not-prose group text-[var(--text-primary)] no-underline' target='_blank' rel='noopener noreferrer' - onClick={(event) => handleLinkClick(event, href)} + onClick={(event) => handleExternalLinkClick(event, href)} > ({ + shouldOpenInBrowserPanel: () => false, + openInBrowserPanel: vi.fn(), +})) +vi.mock('@/lib/integrations', () => ({ + blockTypeToIconMap: { confluence_v2: () => }, +})) + +import { MessageSources } from '@/app/workspace/[workspaceId]/home/components/message-content/components/message-sources/message-sources' + +let root: Root | null = null +let container: HTMLDivElement | null = null + +function mount(ui: React.ReactNode) { + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) + act(() => root?.render(ui)) +} + +function trigger(): HTMLButtonElement { + const node = container?.querySelector('button') + if (!node) throw new Error('Sources button did not render') + return node +} + +/** Opens the popover the way a pointer does — Radix opens on `pointerdown` then `click`. */ +function open() { + act(() => { + trigger().dispatchEvent(new MouseEvent('pointerdown', { bubbles: true, button: 0 })) + trigger().dispatchEvent(new MouseEvent('click', { bubbles: true, button: 0 })) + }) +} + +function links(): HTMLAnchorElement[] { + return Array.from(document.querySelectorAll('a[data-source-link]')) +} + +afterEach(() => { + if (root) act(() => root?.unmount()) + container?.remove() + root = null + container = null +}) + +describe('MessageSources', () => { + it('renders one counted button and lists every source once opened', () => { + mount( + + ) + + expect(trigger().getAttribute('aria-label')).toBe('2 sources') + expect(trigger().textContent).toBe('2') + expect(links()).toHaveLength(0) + + open() + + expect(links().map((link) => link.textContent)).toEqual(['Docs A', 'example.com']) + expect(links().map((link) => link.getAttribute('href'))).toEqual([ + 'https://docs.github.com/en/a', + 'https://www.example.com/page', + ]) + expect(links()[0].getAttribute('target')).toBe('_blank') + }) + + it('shows the connector brand mark when the source names a connector, else the favicon', () => { + mount( + + ) + open() + + const rows = Array.from(document.querySelectorAll('[data-source-link]')).map( + (link) => link.parentElement as HTMLElement + ) + expect(rows[0].querySelector('svg[data-brand="confluence"]')).not.toBeNull() + expect(rows[0].querySelector('img')).toBeNull() + expect(rows[1].querySelector('img')?.getAttribute('src')).toContain('docs.github.com') + }) + + it('renders nothing without sources', () => { + mount() + expect(container?.querySelector('button')).toBeNull() + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/message-sources/message-sources.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/message-sources/message-sources.tsx new file mode 100644 index 00000000000..5b32e024ed3 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/message-sources/message-sources.tsx @@ -0,0 +1,48 @@ +'use client' + +import { Popover, PopoverContent, PopoverTrigger, Tooltip } from '@sim/emcn' +import { BookOpen } from '@sim/emcn/icons' +import { SourceCard } from '@/app/workspace/[workspaceId]/home/components/message-content/components/source-card' +import type { SourceTagData } from '@/app/workspace/[workspaceId]/home/components/message-content/components/special-tags' + +/** The action-row button, matching the copy and vote buttons beside it with room for a count. */ +const BUTTON_CLASSES = + 'flex h-[26px] items-center gap-1 rounded-[6px] px-1.5 text-[var(--text-icon)] text-caption transition-colors hover-hover:bg-[var(--surface-hover)] focus-visible:outline-none data-[state=open]:bg-[var(--surface-active)] data-[state=open]:hover-hover:bg-[var(--surface-active)]' + +interface MessageSourcesProps { + sources: readonly SourceTagData[] +} + +/** + * The documents a reply cited, once each, behind one button in the reply's + * action row: the prose already cites each claim inline, so the full list is + * there for whoever wants it without a second block under the answer. Opens a + * popover of one dense row per document. + */ +export function MessageSources({ sources }: MessageSourcesProps) { + if (sources.length === 0) return null + const label = `${sources.length} ${sources.length === 1 ? 'source' : 'sources'}` + + return ( + + + + + + + + {label} + + +
+ {sources.map((source) => ( + + ))} +
+ + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/index.ts new file mode 100644 index 00000000000..2fe0097f08e --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/index.ts @@ -0,0 +1,6 @@ +export { + highlightTerms, + SOURCE_ROW_CLASSES, + SOURCE_ROW_MARK_CLASSES, + SourceCard, +} from './source-card' diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx new file mode 100644 index 00000000000..4dcf591b77e --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-card/source-card.tsx @@ -0,0 +1,219 @@ +'use client' + +import { type ReactNode, useState } from 'react' +import { Button, chipIconSlotClass, cn, OverflowText, Tooltip } from '@sim/emcn' +import { Check, Link as LinkIcon } from '@sim/emcn/icons' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { formatDate } from '@sim/utils/formatting' +import { faviconUrl } from '@/lib/core/utils/favicon' +import { findTermMatches, queryTerms } from '@/lib/knowledge/search/snippet' +import { + externalLinkHostname, + handleExternalLinkClick, + hideBrokenFavicon, +} from '@/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/external-link' +import { + BRAND_ICON_BY_BASE_TYPE, + sourceLabel, +} from '@/app/workspace/[workspaceId]/home/components/message-content/components/source-chip' +import type { SourceTagData } from '@/app/workspace/[workspaceId]/home/components/message-content/components/special-tags' +import { BrandIcon } from '@/blocks/brand-icon' + +const logger = createLogger('SourceCard') + +/** How long the copied state shows on the copy-link action. */ +const COPIED_FEEDBACK_MS = 1_500 + +/** + * The row every source card and its linkless sibling share: the chat surface's + * row rhythm, a hairline between adjacent rows, and the surface fill on hover + * or focus, so a list of results reads like the lists around the composer. + */ +export const SOURCE_ROW_CLASSES = + 'group/source not-prose flex items-start gap-2 border-[var(--border)] px-2 py-2 transition-colors focus-within:bg-[var(--surface-5)] hover-hover:bg-[var(--surface-5)] [&+&]:border-t' + +/** The 16px mark slot, nudged to centre on the title's first line. */ +export const SOURCE_ROW_MARK_CLASSES = cn(chipIconSlotClass, 'mt-[3px]') + +/** + * The snippet with every query term in bold, so the reader sees why the + * document matched. Terms are matched as whole words in any script, + * case-insensitively, by the same rule the snippet was centred with. + */ +export function highlightTerms(text: string, query: string | undefined): ReactNode { + const matches = findTermMatches(text, queryTerms(query)) + if (matches.length === 0) return text + const parts: ReactNode[] = [] + let cursor = 0 + for (const match of matches) { + if (match.index > cursor) parts.push(text.slice(cursor, match.index)) + parts.push( + + {text.slice(match.index, match.index + match.length)} + + ) + cursor = match.index + match.length + } + if (cursor < text.length) parts.push(text.slice(cursor)) + return parts +} + +function parseUpdatedAt(value: string | undefined): Date | null { + if (!value) return null + const date = new Date(value) + return Number.isNaN(date.getTime()) ? null : date +} + +interface CopyLinkActionProps { + url: string +} + +/** + * Copies the document's link; confirms with a check for a moment. The check + * only shows once the clipboard accepted the write: a page denied clipboard + * access is left at "Copy link" rather than claiming a copy that never landed. + */ +function CopyLinkAction({ url }: CopyLinkActionProps) { + const [copied, setCopied] = useState(false) + return ( + + + + + {copied ? 'Copied' : 'Copy link'} + + ) +} + +interface SourceCardProps { + source: SourceTagData + /** The query the document was found for; its terms are bolded in the snippet. */ + query?: string + /** Offers a Summarize action that asks the agent about this document. */ + onSummarize?: (source: SourceTagData) => void + /** + * One line per document: the mark, the title, and where it lives, with no + * snippet. For a list under a reply whose prose already cites each claim. + */ + dense?: boolean +} + +/** + * One document a search found, laid out to be scanned: the source's brand + * mark or favicon, the title as a link back to the document, where it lives, + * who it is from, and when it last changed, and the passage that matched with + * the query terms in bold. Actions stay out of the way until the row is + * hovered or its title focused. The same row serves the composer's search + * results and, in its dense form, the footer of a reply that cited sources. + */ +export function SourceCard({ source, query, onSummarize, dense = false }: SourceCardProps) { + const hostname = externalLinkHostname(source.url) + const ConnectorIcon = source.connectorType + ? BRAND_ICON_BY_BASE_TYPE.get(source.connectorType) + : undefined + const updatedAt = parseUpdatedAt(source.updatedAt) + const meta = [ + sourceLabel(source), + source.author?.trim() || null, + updatedAt ? formatDate(updatedAt) : null, + ].filter((part): part is string => Boolean(part)) + + const mark = ConnectorIcon ? ( + + ) : hostname ? ( + + ) : null + + if (dense) { + return ( + + ) + } + + return ( +
+ {mark} + +
+ + {onSummarize && ( + + )} +
+
+ ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/index.ts new file mode 100644 index 00000000000..329fa2848eb --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/index.ts @@ -0,0 +1 @@ +export { BRAND_ICON_BY_BASE_TYPE, SourceChip, sourceLabel } from './source-chip' diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/source-chip.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/source-chip.tsx new file mode 100644 index 00000000000..03d00442aa1 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/source-chip/source-chip.tsx @@ -0,0 +1,89 @@ +'use client' + +import { chipFilledFillTokens, chipHoverSurfaceClass, cn, OverflowText, Tooltip } from '@sim/emcn' +import { stripVersionSuffix } from '@sim/utils/string' +import { faviconUrl } from '@/lib/core/utils/favicon' +import { blockTypeToIconMap } from '@/lib/integrations' +import { + externalLinkHostname, + handleExternalLinkClick, + hideBrokenFavicon, +} from '@/app/workspace/[workspaceId]/home/components/message-content/components/chat-content/external-link' +import type { SourceTagData } from '@/app/workspace/[workspaceId]/home/components/message-content/components/special-tags' +import { BrandIcon, type StyleableIcon } from '@/blocks/brand-icon' + +/** + * Brand marks by base block type. A connector id names the same product as its + * integration block (`confluence`, `google_drive`), so the block's mark serves + * the chip — through the catalog icon map rather than the connector registry, + * which would drag seventy connector modules into every surface that renders + * chat. Versioned catalog types (`gmail_v2`) collapse onto their base name. + */ +export const BRAND_ICON_BY_BASE_TYPE: ReadonlyMap = new Map( + Object.entries(blockTypeToIconMap).map(([type, icon]) => [stripVersionSuffix(type), icon]) +) + +/** Chip label: the site name the model supplied, else the URL's hostname without a `www.` prefix. */ +export function sourceLabel(source: SourceTagData): string { + const siteName = source.siteName?.trim() + if (siteName) return siteName + return (externalLinkHostname(source.url) ?? source.url).replace(/^www\./, '') +} + +interface SourceChipProps { + source: SourceTagData +} + +/** + * A cited document as a small round pill — the connector's brand mark or the + * site favicon, then the site name — used inline at the citation point and + * again in the footer strip. Built on the chip fill and hover tokens at a 20px + * height so it sits inside a line of prose; the 30px `Chip` is the wrong scale + * for a citation. Opens the document like any external link in the reply. + */ +export function SourceChip({ source }: SourceChipProps) { + const hostname = externalLinkHostname(source.url) + const ConnectorIcon = source.connectorType + ? BRAND_ICON_BY_BASE_TYPE.get(source.connectorType) + : undefined + + return ( + + + handleExternalLinkClick(event, source.url)} + className={cn( + 'not-prose inline-flex h-[20px] max-w-[220px] shrink-0 items-center gap-1 rounded-full px-1.5 align-middle text-[var(--text-body)] text-caption no-underline transition-colors', + chipFilledFillTokens, + chipHoverSurfaceClass + )} + > + {ConnectorIcon ? ( + + ) : hostname ? ( + + ) : null} + + + + + {source.title ? ( + + {source.title} + {source.url} + + ) : ( + {source.url} + )} + + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/index.ts index 1603a0278c5..609cdc21eca 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/index.ts @@ -13,6 +13,7 @@ export type { QuestionTagData, QuestionType, RuntimeSpecialTagName, + SourceTagData, UsageUpgradeAction, UsageUpgradeTagData, WorkspaceResourceTagData, @@ -24,6 +25,7 @@ export { CredentialDisplay, credentialTagHasVisibleCard, formatCredentialSubmissionMessage, + isHttpUrl, PendingTagIndicator, parseCredentialSubmissionMessage, parseCredentialSubmissionProgress, diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts index 5063faecdba..3dab938af28 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/components/special-tags/special-tags.test.ts @@ -1073,6 +1073,7 @@ describe('parser properties', () => { '{"reason":"monthly cap","action":"upgrade_plan","message":"You hit your limit."}', 'mothership-error': '{"message":"The tool call failed.","code":"E_TOOL"}', + source: '{"url":"https://docs.github.com/en/x","siteName":"GitHub Docs"}', } /** Renders nothing rather than a card, so it cannot carry a card invariant. */ @@ -1374,3 +1375,52 @@ describe('ordinary JSON in prose is never turned into a card', () => { expect(segments.some((s) => s.type === 'text')).toBe(true) }) }) + +describe('source tag', () => { + it('parses a complete source tag into a source segment', () => { + const { segments } = parseSpecialTags( + 'Remove them first. {"url":"https://docs.github.com/en/x","siteName":"GitHub Docs","title":"Blocking users"} Then block.', + false + ) + + expect(segments).toEqual([ + { type: 'text', content: 'Remove them first. ' }, + { + type: 'source', + data: { + url: 'https://docs.github.com/en/x', + siteName: 'GitHub Docs', + title: 'Blocking users', + }, + }, + { type: 'text', content: ' Then block.' }, + ]) + }) + + it('keeps adjacent source tags as separate segments', () => { + const { segments } = parseSpecialTags( + 'Done. {"url":"https://a.example/1"}{"url":"https://b.example/2"}', + false + ) + + expect(segments.filter((segment) => segment.type === 'source')).toHaveLength(2) + }) + + it('rejects a source without an absolute http(s) url', () => { + for (const url of ['docs/internal.md', 'https://?', 'ftp://host/x', 'https://a b.example/x']) { + const { segments } = parseSpecialTags( + `See {"url":"${url}","siteName":"Docs"}.`, + false + ) + + expect(segments.some((segment) => segment.type === 'source')).toBe(false) + } + }) + + it('hides a half-arrived source opener while streaming', () => { + const { segments, hasPendingTag } = parseSpecialTags('Block them. ` tag: one document the reply drew on. The tag contract for + * search answers — the model emits it inline, right after the sentence, list + * item, or paragraph the document supports, as a JSON body: + * + * `{"url":"https://docs.github.com/…","siteName":"GitHub Docs"}` + * + * Each tag renders as its own small chip where it sits (adjacent tags are + * never collapsed into a count), and every distinct `url` in the message is + * repeated in the footer strip below the reply. + */ +export interface SourceTagData { + /** Canonical http(s) link to the referenced document. */ + url: string + /** Document title, shown on hover. */ + title?: string + /** + * Short chip label — the site or product the document lives in ("GitHub + * Docs", "Confluence"). Falls back to the URL's hostname. + */ + siteName?: string + /** + * Knowledge-base connector the document was synced through + * (`CONNECTOR_META_REGISTRY` key). Lends the chip the product's brand mark; + * without it the chip shows the site favicon. + */ + connectorType?: string + /** The passage the reply relied on; a reply whose sources carry one is listed as result cards. */ + snippet?: string + /** When the source last changed the document, as an ISO timestamp. */ + updatedAt?: string + /** The person behind the document, as the source names them. */ + author?: string +} + export type ContentSegment = | { type: 'text'; content: string } | { type: 'thinking'; content: string } @@ -325,6 +360,7 @@ export type ContentSegment = | { type: 'mothership-error'; data: MothershipErrorTagData } | { type: 'workspace_resource'; data: WorkspaceResourceTagData } | { type: 'question'; data: QuestionTagData } + | { type: 'source'; data: SourceTagData } export type RuntimeSpecialTagName = | 'thinking' @@ -334,6 +370,7 @@ export type RuntimeSpecialTagName = | 'file' | 'workspace_resource' | 'question' + | 'source' export interface ParsedSpecialContent { segments: ContentSegment[] @@ -348,6 +385,7 @@ const RUNTIME_SPECIAL_TAG_NAMES = [ 'file', 'workspace_resource', 'question', + 'source', ] as const /** @@ -363,6 +401,7 @@ export const SPECIAL_TAG_NAMES = [ 'mothership-error', 'workspace_resource', 'question', + 'source', ] as const function isOptionsItemData(value: unknown): value is OptionsItemData { @@ -523,6 +562,33 @@ function isMothershipErrorTagData(value: unknown): value is MothershipErrorTagDa ) } +/** + * Only an absolute http(s) URL with a host can be linked; anything else is not + * a source. Parsed rather than pattern-matched so a malformed value such as + * `https://?` — which a prefix check would accept — never becomes a dead link. + */ +export function isHttpUrl(value: unknown): value is string { + if (typeof value !== 'string' || /\s/.test(value)) return false + try { + const url = new URL(value) + return (url.protocol === 'http:' || url.protocol === 'https:') && url.hostname.length > 0 + } catch { + return false + } +} + +function isSourceTagData(value: unknown): value is SourceTagData { + if (!isRecordLike(value)) return false + if (!isHttpUrl(value.url)) return false + if (value.title !== undefined && typeof value.title !== 'string') return false + if (value.siteName !== undefined && typeof value.siteName !== 'string') return false + if (value.connectorType !== undefined && typeof value.connectorType !== 'string') return false + if (value.snippet !== undefined && typeof value.snippet !== 'string') return false + if (value.updatedAt !== undefined && typeof value.updatedAt !== 'string') return false + if (value.author !== undefined && typeof value.author !== 'string') return false + return true +} + function isWorkspaceResourceTagData(value: unknown): value is WorkspaceResourceTagData { if (!isRecordLike(value)) return false if ( @@ -713,6 +779,7 @@ function parseSpecialTagData( | { type: 'mothership-error'; data: MothershipErrorTagData } | { type: 'workspace_resource'; data: WorkspaceResourceTagData } | { type: 'question'; data: QuestionTagData } + | { type: 'source'; data: SourceTagData } | null { if (tagName === 'thinking') { const content = parseTextTagBody(body) @@ -744,6 +811,11 @@ function parseSpecialTagData( return data ? { type: 'workspace_resource', data } : null } + if (tagName === 'source') { + const data = parseJsonTagBody(body, isSourceTagData) + return data ? { type: 'source', data } : null + } + if (tagName === 'question') { const data = parseQuestionTagBody(body) if (data) return { type: 'question', data } @@ -1659,7 +1731,8 @@ interface SpecialTagsProps { /** * Unified renderer for inline special tags: ``, ``, ``, - * and ``. + * and ``. A `` never reaches here — the chat renderer + * folds it into the surrounding markdown as an inline chip. */ export function SpecialTags({ segment, @@ -1692,6 +1765,8 @@ export function SpecialTags({ return case 'workspace_resource': return + case 'source': + return null case 'question': return ( 0 - ? parsed - : fallbackContent?.trim() - ? [{ type: 'text' as const, id: 'text-fallback', content: fallbackContent }] - : [] + const segments = useMemo( + () => + parsed.length > 0 + ? parsed + : fallbackContent?.trim() + ? [{ type: 'text', id: 'text-fallback', content: fallbackContent }] + : [], + [parsed, fallbackContent] + ) + /** + * Collected from the segments that render, not the raw blocks: that is the + * same text the inline chips come from, so the footer agrees with them — it + * covers the fallback text of a block-less message and leaves out lane text + * that `parseBlocks` folds into agent groups. + */ + const sources = useMemo( + () => + collectMessageSources( + segments.flatMap((segment) => (segment.type === 'text' ? [segment.content] : [])) + ), + [segments] + ) const visibleStreamActivityKey = getVisibleStreamActivityKey(segments) // Every visible stream update restarts the quiet-period clock. A layout @@ -935,6 +958,13 @@ function MessageContentInner({ trailingPendingTag || (isStreamIdle && !trailingStreamActivity && !hasExecutingTool)) + const actionsRow = ( +
+ {actions} + {sources.length > 0 && } +
+ ) + return (
@@ -1032,10 +1062,10 @@ function MessageContentInner({ Stopped by user
- {actions &&
{actions}
} + {actions &&
{actionsRow}
} ) : ( - actions &&
{actions}
+ actions &&
{actionsRow}
)}
) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/utils.test.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/utils.test.ts index f0a74e539a7..de5bd6402fe 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/utils.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/utils.test.ts @@ -2,7 +2,7 @@ * @vitest-environment node */ import { describe, expect, it } from 'vitest' -import { deriveMessagePhase, resolveToolDisplayState } from './utils' +import { collectMessageSources, deriveMessagePhase, resolveToolDisplayState } from './utils' describe('deriveMessagePhase', () => { it('is streaming whenever the transport is live', () => { @@ -36,3 +36,25 @@ describe('resolveToolDisplayState', () => { expect(resolveToolDisplayState('rejected')).toBe('icon') }) }) + +describe('collectMessageSources', () => { + const source = (url: string, extra = '') => `{"url":"${url}"${extra}}` + + it('collects every distinct source across the given text, in first-cited order', () => { + const texts = [ + `First point. ${source('https://a.example/1', ',"siteName":"A"')} Second. ${source('https://b.example/2')}`, + `Again. ${source('https://a.example/1')} New. ${source('https://c.example/3')}`, + ] + + expect(collectMessageSources(texts).map((entry) => entry.url)).toEqual([ + 'https://a.example/1', + 'https://b.example/2', + 'https://c.example/3', + ]) + expect(collectMessageSources(texts)[0].siteName).toBe('A') + }) + + it('returns nothing for prose without sources', () => { + expect(collectMessageSources(['Plain prose.', ''])).toEqual([]) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/utils.ts b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/utils.ts index 64176c822c5..166f5fd01c5 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/message-content/utils.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/message-content/utils.ts @@ -21,10 +21,31 @@ import { } from '@sim/emcn' import { Calendar, Clock, Cursor, Globe, Table as TableIcon } from '@sim/emcn/icons' import { AgentIcon, ImageIcon, TTSIcon, VideoIcon } from '@/components/icons' +import { + parseSpecialTags, + type SourceTagData, +} from '@/app/workspace/[workspaceId]/home/components/message-content/components/special-tags' import type { ToolCallStatus } from '@/app/workspace/[workspaceId]/home/types' export type IconComponent = ComponentType> +/** + * Every distinct `` cited across the given prose, in first-cited order, + * for the footer strip. Callers pass the text segments the message actually + * renders as its answer. + */ +export function collectMessageSources(texts: readonly string[]): SourceTagData[] { + const byUrl = new Map() + for (const text of texts) { + for (const segment of parseSpecialTags(text, false).segments) { + if (segment.type === 'source' && !byUrl.has(segment.data.url)) { + byUrl.set(segment.data.url, segment.data) + } + } + } + return [...byUrl.values()] +} + const TOOL_ICONS: Record = { mothership: Blimp, glob: FolderCode, diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/mothership-chat.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/mothership-chat.tsx index f4f948f429f..977ef369b7f 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/mothership-chat.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/mothership-chat/mothership-chat.tsx @@ -2,6 +2,8 @@ import { memo, + type ReactNode, + type RefObject, useCallback, useDeferredValue, useEffect, @@ -60,6 +62,14 @@ interface MothershipChatProps { workspaceId: string messages: ChatMessage[] isSending: boolean + /** The composer's Search-mode results, shown above the input. */ + searchResults?: ReactNode + /** The live search query; the composer shows it so the box and the results never disagree. */ + searchQuery?: string + /** The composer, for a caller that hands a question to the agent from outside the box. */ + userInputRef?: RefObject + /** Puts the composer in the mode a queued message was written in, when one is loaded for editing. */ + onRestoreQueuedMode?: (requestMode: QueuedMessage['requestMode']) => void isReconnecting?: boolean isLoading?: boolean onSubmit: ( @@ -67,6 +77,12 @@ interface MothershipChatProps { fileAttachments?: FileAttachmentForApi[], contexts?: ChatContext[] ) => void + /** Whether the composer offers Search mode; only the Home composer answers a search. */ + canSearch?: boolean + /** Off in Search mode, where the query stays put so the person can refine it. */ + clearOnSubmit?: boolean + /** Fires when the composer's text goes from something to nothing. */ + onCleared?: () => void onStopGeneration: () => void messageQueue: QueuedMessage[] editingQueuedId: string | null @@ -195,7 +211,6 @@ interface AssistantMessageRowProps { prepareContentForCopy: (content: string) => ClipboardContent isStreaming: boolean isLast: boolean - precedingUserContent?: string /** Transcript-derived answers for this message's question card (renders the recap). */ questionAnswers?: string[] /** Transcript-derived status payload for this message's credential card. */ @@ -212,7 +227,6 @@ const AssistantMessageRow = memo(function AssistantMessageRow({ prepareContentForCopy, isStreaming, isLast, - precedingUserContent, questionAnswers, credentialSubmission, credentialAbandoned, @@ -297,7 +311,6 @@ const AssistantMessageRow = memo(function AssistantMessageRow({ getCopyContent={getCopyContent} hasCopyContent={Boolean(getOrchestratorMessageText(blocks, message.content).trim())} prepareContentForCopy={prepareContentForCopy} - userQuery={precedingUserContent} requestId={message.requestId} messageId={message.id} /> @@ -312,9 +325,16 @@ export function MothershipChat({ workspaceId, messages: messagesProp, isSending, + searchResults, + searchQuery, + userInputRef: userInputRefProp, + onRestoreQueuedMode, isReconnecting = false, isLoading = false, onSubmit, + canSearch = false, + clearOnSubmit, + onCleared, onStopGeneration, messageQueue, editingQueuedId, @@ -542,16 +562,6 @@ export function MothershipChat({ return out }, [messages]) - const precedingUserContentByIndex = useMemo(() => { - const out: Array = [] - let lastUserContent: string | undefined - for (const [index, message] of messages.entries()) { - out[index] = lastUserContent - if (message.role === 'user') lastUserContent = message.content - } - return out - }, [messages]) - /** * Pairs each assistant question/credential card with the user message that * completed it. The paired user message is hidden — the answered card IS the @@ -663,7 +673,8 @@ export function MothershipChat({ item.index !== lastIndex && item.start < (instance.scrollElement?.scrollTop ?? 0) const scrolledChatRef = useRef(UNSCROLLED) - const userInputRef = useRef(null) + const ownUserInputRef = useRef(null) + const userInputRef = userInputRefProp ?? ownUserInputRef const messageQueueRef = useRef(messageQueue) useEffect(() => { messageQueueRef.current = messageQueue @@ -686,9 +697,11 @@ export function MothershipChat({ const handleEditQueued = useCallback( (id: string) => { const msg = onEditQueuedMessage(id) - if (msg) userInputRef.current?.loadQueuedMessage(msg) + if (!msg) return + onRestoreQueuedMode?.(msg.requestMode) + userInputRef.current?.loadQueuedMessage(msg) }, - [onEditQueuedMessage] + [onEditQueuedMessage, onRestoreQueuedMode, userInputRef] ) const handleEditQueuedTail = useCallback(() => { @@ -796,7 +809,6 @@ export function MothershipChat({ prepareContentForCopy={prepareContentForCopy} isStreaming={isStreamActive && isLast} isLast={isLast} - precedingUserContent={precedingUserContentByIndex[index]} questionAnswers={interactionPairing.answersByIndex[index]} credentialSubmission={interactionPairing.credentialSubmissionByIndex[index]} credentialAbandoned={interactionPairing.credentialAbandonedByIndex[index]} @@ -817,6 +829,9 @@ export function MothershipChat({ onAnimationEnd={animateInput ? onInputAnimationEnd : undefined} >
+ {searchResults && ( +
{searchResults}
+ )} + canConnectPersonally(connector.meta) +) + +/** The Sim Search connection per source, keyed by connector type. */ +function simSearchConnectionsByType( + connectors: readonly WorkspaceMemberConnector[] +): Map { + const byType = new Map() + for (const connector of connectors) { + if (connector.knowledgeBaseName !== SIM_SEARCH_KNOWLEDGE_BASE_NAME) continue + if (!byType.has(connector.connectorType)) byType.set(connector.connectorType, connector) + } + return byType +} + +/** Whether a connected source is still indexing for the viewer. */ +export function isIndexing(connection: WorkspaceMemberConnector | undefined): boolean { + return ( + connection?.viewerMembership === 'connected' && + (connection.memberSyncStatus === 'pending' || connection.memberSyncStatus === 'running') + ) +} + +/** The chip's trailing state text for one source. */ +function sourceState( + connection: WorkspaceMemberConnector | undefined, + waiting: boolean +): string | null { + if (waiting) return 'Connecting…' + if (!connection) return null + switch (connection.viewerMembership) { + case 'connected': + return isIndexing(connection) + ? 'Indexing' + : connection.viewerDocumentCount === 1 + ? '1 document' + : `${connection.viewerDocumentCount} documents` + case 'needs_reauth': + return 'Reconnect' + case 'unverified_email': + return 'Verify email' + case 'revoked': + return 'Access removed' + default: + return null + } +} + +interface SourceChipProps { + connector: SearchConnector + connection: WorkspaceMemberConnector | undefined + /** Why the source cannot be connected here, shown as the chip's title; null when it can. */ + unavailableReason: string | null + waiting: boolean + disabled: boolean + onConnect: () => void +} + +function SourceChip({ + connector, + connection, + unavailableReason, + waiting, + disabled, + onConnect, +}: SourceChipProps) { + const state = sourceState(connection, waiting) + const connected = connection?.viewerMembership === 'connected' + const unavailable = unavailableReason !== null + const actionable = + !unavailable && + !waiting && + (!connection || CONNECTABLE_MEMBERSHIPS.has(connection.viewerMembership)) + const title = + unavailableReason ?? + (connected ? `${connector.meta.name}: ${state}` : `Connect ${connector.meta.name}`) + const busy = waiting || isIndexing(connection) + return ( + } + rightIcon={!busy && actionable ? Plus : undefined} + rightAdornment={ + busy ? : undefined + } + > + + {connector.meta.name} + {state && {state}} + + + ) +} + +interface SearchSourcesProps { + workspaceId: string +} + +/** + * Every source a person can connect themselves, as chips under the composer: + * connected ones show how many documents they can read (or that indexing is + * still running), the rest connect with one click. A source that needs a site + * or space asks for it once, in place, on the connect that creates it; + * everyone after that clicks straight through. Sources an admin must set up + * as workspace connectors do not appear here. + */ +export function SearchSources({ workspaceId }: SearchSourcesProps) { + const { integrationAvailability } = usePermissionConfig() + const { features } = useWorkspaceHostContext() + /** + * Judged by the workspace, as the server judges it: with per-member access + * off, a connect is refused, so the chips say so instead of offering one. + */ + const memberAccessAvailable = features?.knowledgeMemberAccess === true + const { data: workspacePermissions } = useWorkspacePermissionsQuery(workspaceId) + /** The first connect of a source turns it on for the workspace, which takes an admin. */ + const canCreate = workspacePermissions?.viewer?.isAdmin ?? false + const { data: memberConnectorRows } = useWorkspaceMemberConnectors(workspaceId, { + enabled: memberAccessAvailable, + }) + /** Rows cached before the feature went off are not this surface's to show. */ + const memberConnectors = memberAccessAvailable + ? (memberConnectorRows ?? EMPTY_MEMBER_CONNECTORS) + : EMPTY_MEMBER_CONNECTORS + const connectionByType = useMemo( + () => simSearchConnectionsByType(memberConnectors), + [memberConnectors] + ) + const connectedConnectorIds = useMemo( + () => + new Set( + memberConnectors + .filter((connector) => connector.viewerMembership === 'connected') + .map((connector) => connector.connectorId) + ), + [memberConnectors] + ) + const membershipQueryKeys = useMemo(() => [memberConnectorKeys.list(workspaceId)], [workspaceId]) + const { + connectSource, + connectSearchSource, + setupConnector, + closeSetup, + isAwaiting, + isAwaitingSource, + isPending, + error, + } = useMemberEnrollment({ membershipQueryKeys, connectedConnectorIds }) + + /** Connected sources first; the catalog is already alphabetical, so the partition keeps the order. */ + const isConnected = (connector: SearchConnector) => + connectionByType.get(connector.type)?.viewerMembership === 'connected' + const ordered = [ + ...PERSONAL_SEARCH_CONNECTORS.filter(isConnected), + ...PERSONAL_SEARCH_CONNECTORS.filter((connector) => !isConnected(connector)), + ] + + return ( +
+
+ {ordered.map((connector) => { + const connection = connectionByType.get(connector.type) + return ( + connectSearchSource(workspaceId, connector, connection)} + /> + ) + })} +
+ {error &&

{error}

} + {setupConnector && ( + + connectSource(workspaceId, setupConnector.type, sourceConfig) + } + /> + )} +
+ ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal.tsx new file mode 100644 index 00000000000..3537ebec9ed --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal.tsx @@ -0,0 +1,82 @@ +'use client' + +import { useState } from 'react' +import { + ChipModal, + ChipModalBody, + ChipModalField, + ChipModalFooter, + ChipModalHeader, +} from '@sim/emcn' +import type { SearchConnector } from '@/lib/sim-search/connectors' + +interface SourceSetupModalProps { + connector: SearchConnector + onClose: () => void + /** Connects the source with the filled-in fields; the caller opens the OAuth tab in this click. */ + onConnect: (sourceConfig: Record) => void +} + +/** + * The few fields a source needs before its first connect, such as a site and + * a space. Everyone after the first person clicks straight through. + */ +export function SourceSetupModal({ connector, onClose, onConnect }: SourceSetupModalProps) { + const fields = connector.setupFields + const [values, setValues] = useState>({}) + const complete = fields.every((field) => values[field.id]?.trim()) + + const submit = () => { + if (!complete) return + onConnect(Object.fromEntries(fields.map((field) => [field.id, values[field.id]?.trim() ?? '']))) + onClose() + } + + return ( + { + if (!open) onClose() + }} + srTitle={`Connect ${connector.meta.name}`} + > + Connect {connector.meta.name} + + {fields.map((field) => + field.type === 'dropdown' ? ( + setValues((current) => ({ ...current, [field.id]: value }))} + options={(field.options ?? []).map((option) => ({ + value: option.id, + label: option.label, + }))} + placeholder={field.placeholder} + hint={field.description} + required + /> + ) : ( + setValues((current) => ({ ...current, [field.id]: value }))} + placeholder={field.placeholder} + hint={field.description} + autoComplete='off' + required + /> + ) + )} + + + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx new file mode 100644 index 00000000000..a879ee5bc35 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.test.tsx @@ -0,0 +1,153 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockCaptureEvent, modeState } = vi.hoisted(() => ({ + mockCaptureEvent: vi.fn(), + /** The URL `mode` param as the nuqs mock serves it; `set` is the live setter once mounted. */ + modeState: { initial: 'build', set: (_next: string) => {} }, +})) + +vi.mock('nuqs', async () => { + const { useState } = await import('react') + return { + useQueryState: () => { + const [mode, setMode] = useState(modeState.initial) + modeState.set = setMode + return [mode, setMode] + }, + } +}) + +vi.mock('next/navigation', () => ({ + useParams: () => ({ workspaceId: 'workspace-1' }), +})) +vi.mock('posthog-js/react', () => ({ usePostHog: () => null })) +vi.mock('@/lib/posthog/client', () => ({ captureEvent: mockCaptureEvent })) +vi.mock('@sim/utils/random', () => ({ randomFloat: () => 0 })) + +vi.mock('@/hooks/queries/credentials', () => ({ + useWorkspaceCredentials: () => ({ data: [] }), +})) +vi.mock('@/hooks/queries/oauth/oauth-connections', () => ({ + useOAuthConnections: () => ({ data: [] }), +})) +vi.mock('@/hooks/queries/tables', () => ({ + useTablesList: () => ({ data: [] }), +})) +vi.mock('@/hooks/queries/kb/knowledge', () => ({ + useKnowledgeBasesQuery: () => ({ data: [] }), +})) +vi.mock('@/app/workspace/[workspaceId]/home/components/search-sources', () => ({ + SearchSources: () =>
, +})) +vi.mock('@/hooks/use-permission-config', () => ({ + usePermissionConfig: () => ({ + integrationAvailability: new Map([['notion', { state: 'unavailable', oauthAvailable: false }]]), + }), +})) + +/** The Build-mode pool is built from the block catalog at module load; an empty catalog keeps it to the table starters. */ +vi.mock('@/blocks/registry', () => ({ getAllBlockMeta: () => ({}), getAllBlocks: () => [] })) + +vi.mock('@/lib/sim-search/connectors', () => { + const icon = () => null + const connector = (type: string, name: string, providerId: string) => ({ + type, + meta: { id: type, name, description: `Sync ${name}`, icon }, + providerId, + providerIds: [providerId], + requiredScopes: ['read'], + serviceName: name, + serviceIcon: icon, + blockType: type, + }) + return { + isSearchConnectorAvailable: ( + candidate: { blockType: string }, + availability: ReadonlyMap + ) => availability.get(candidate.blockType)?.oauthAvailable ?? true, + SEARCH_CONNECTORS: [ + connector('airtable', 'Airtable', 'airtable'), + connector('confluence', 'Confluence', 'confluence'), + connector('jira', 'Jira', 'jira'), + connector('jsm', 'Jira Service Management', 'jira'), + connector('notion', 'Notion', 'notion'), + connector('slack', 'Slack', 'slack'), + ], + } +}) + +vi.mock('@/app/workspace/[workspaceId]/components/connect-oauth-modal', () => ({ + ConnectOAuthModal: ({ open, providerId }: { open: boolean; providerId: string }) => + open ?
{providerId}
: null, +})) + +import { SuggestedActions } from '@/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions' + +let root: Root | null = null +let container: HTMLDivElement | null = null +const onSelectPrompt = vi.fn() + +function mount() { + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) + act(() => root?.render()) +} + +function heading(): string { + return container?.querySelector('button[aria-expanded] span')?.textContent ?? '' +} + +function rows(): HTMLButtonElement[] { + return Array.from( + container?.querySelectorAll('button:not([aria-expanded])') ?? [] + ) +} + +beforeEach(() => { + onSelectPrompt.mockClear() + mockCaptureEvent.mockClear() + modeState.initial = 'build' +}) + +afterEach(() => { + if (root) act(() => root?.unmount()) + container?.remove() + root = null + container = null +}) + +describe('SuggestedActions', () => { + it('shows the Build starters by default', () => { + mount() + + expect(heading()).toBe('Suggested actions') + expect(rows().map((row) => row.textContent)).toContain('Integrate with Slack') + }) + + it('shows every source in Search mode instead of the sampled suggestions', () => { + mount() + + act(() => modeState.set('search')) + + expect(heading()).toBe('Sources') + expect(document.querySelector('[data-testid="search-sources"]')).not.toBeNull() + expect(rows()).toHaveLength(0) + }) + + it('shows the sources in Assistant mode, which answers from them', () => { + mount() + + act(() => modeState.set('assistant')) + + expect(heading()).toBe('Sources') + expect(document.querySelector('[data-testid="search-sources"]')).not.toBeNull() + expect(rows()).toHaveLength(0) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx index 32f878c1a27..76e1ec28c28 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/suggested-actions.tsx @@ -1,21 +1,28 @@ 'use client' -import { type ComponentType, type CSSProperties, useMemo, useState } from 'react' +import { useMemo, useState } from 'react' import { ArrowRight, ChevronDown, cn, Expandable, ExpandableContent } from '@sim/emcn' import { Table } from '@sim/emcn/icons' -import { randomFloat } from '@sim/utils/random' import { stripVersionSuffix } from '@sim/utils/string' import { useParams } from 'next/navigation' import { usePostHog } from 'posthog-js/react' import { GmailIcon, SlackIcon } from '@/components/icons' import { INTEGRATIONS, - type OAuthServiceMatch, resolveOAuthServiceForIntegration, resolveOAuthServiceForSlug, } from '@/lib/integrations' import { captureEvent } from '@/lib/posthog/client' import { ConnectOAuthModal } from '@/app/workspace/[workspaceId]/components/connect-oauth-modal' +import { SearchSources } from '@/app/workspace/[workspaceId]/home/components/search-sources' +import type { + Action, + ActionIcon, + OAuthConnectTarget, +} from '@/app/workspace/[workspaceId]/home/components/suggested-actions/types' +import { weightedSample } from '@/app/workspace/[workspaceId]/home/components/suggested-actions/weighted-sample' +import { useMothershipMode } from '@/app/workspace/[workspaceId]/home/hooks/use-mothership-mode' +import type { MothershipMode } from '@/app/workspace/[workspaceId]/home/search-params' import { BrandIcon } from '@/blocks/brand-icon' import { getAllBlockMeta } from '@/blocks/registry' import type { ModuleTag } from '@/blocks/types' @@ -23,12 +30,7 @@ import { useWorkspaceCredentials } from '@/hooks/queries/credentials' import { useKnowledgeBasesQuery } from '@/hooks/queries/kb/knowledge' import { useOAuthConnections } from '@/hooks/queries/oauth/oauth-connections' import { useTablesList } from '@/hooks/queries/tables' - -type Icon = ComponentType<{ className?: string; style?: CSSProperties }> - -type Action = - | { kind: 'prompt'; id: string; label: string; prompt: string; icon: Icon } - | { kind: 'integration'; id: string; label: string; icon: Icon; slug: string } +import { usePermissionConfig } from '@/hooks/use-permission-config' /** Lookup integration slug by OAuth service display name (case-insensitive). */ const SLUG_BY_LOWER_NAME: ReadonlyMap = new Map( @@ -51,7 +53,7 @@ interface Candidate { blockType: string label: string prompt: string - icon: Icon + icon: ActionIcon modules: readonly ModuleTag[] featured: boolean popular: boolean @@ -101,7 +103,7 @@ const CANDIDATES: readonly Candidate[] = (() => { blockType, label: template.title, prompt: template.prompt, - icon: template.icon as Icon, + icon: template.icon as ActionIcon, modules: template.modules, featured: template.featured ?? false, popular: template.category === 'popular', @@ -147,34 +149,13 @@ function scoreCandidate(c: Candidate, signals: Signals): number { return weight } -/** - * Weighted sampling without replacement. Each pick's probability is - * proportional to its weight, so the set stays varied while staying relevant. - */ -function weightedSample(pool: readonly T[], n: number, weightOf: (item: T) => number): T[] { - const remaining = pool.map((item) => ({ item, weight: Math.max(weightOf(item), 0) })) - const out: T[] = [] - while (out.length < n && remaining.length > 0) { - const total = remaining.reduce((sum, entry) => sum + entry.weight, 0) - if (total <= 0) break - let roll = randomFloat() * total - const index = remaining.findIndex((entry) => { - roll -= entry.weight - return roll <= 0 - }) - const [picked] = remaining.splice(index === -1 ? remaining.length - 1 : index, 1) - out.push(picked.item) - } - return out -} - const EMPTY_CREDENTIALS: NonNullable['data']> = [] const EMPTY_SERVICES: NonNullable['data']> = [] type ServiceInfo = NonNullable['data']>[number] function toPromptAction(c: Candidate): Action { - return { kind: 'prompt', id: c.id, label: c.label, prompt: c.prompt, icon: c.icon } + return { kind: 'prompt', id: c.id, label: c.label, icon: c.icon, prompt: c.prompt } } function toIntegrationAction(service: ServiceInfo, slug: string): Action { @@ -251,6 +232,13 @@ const INITIAL_ACTIONS: Action[] = [ .map(toPromptAction), ] +/** Section heading per composer mode — Search reads as a connect-your-sources list. */ +const HEADINGS: Record = { + build: 'Suggested actions', + search: 'Sources', + assistant: 'Sources', +} + interface SuggestedActionsProps { onSelectPrompt: (prompt: string) => void } @@ -258,6 +246,8 @@ interface SuggestedActionsProps { export function SuggestedActions({ onSelectPrompt }: SuggestedActionsProps) { const { workspaceId } = useParams<{ workspaceId: string }>() const posthog = usePostHog() + const [mode] = useMothershipMode() + const { integrationAvailability } = usePermissionConfig() const { data: credentials = EMPTY_CREDENTIALS } = useWorkspaceCredentials({ workspaceId, @@ -282,7 +272,7 @@ export function SuggestedActions({ onSelectPrompt }: SuggestedActionsProps) { * to `null` (via `onOpenChange(false)`) closes it. Mirrors the local-state * pattern used by the integrations detail page. */ - const [oauthTarget, setOAuthTarget] = useState(null) + const [oauthTarget, setOAuthTarget] = useState(null) const connectedProviders = useMemo( () => @@ -305,16 +295,24 @@ export function SuggestedActions({ onSelectPrompt }: SuggestedActionsProps) { ) /** - * Personalized suggestions, re-sampled whenever signals resolve. Falls back to + * Each mode's list is memoized on its own inputs alone, so switching modes — + * or the other mode's signals settling — never re-samples it. + * + * Search lists connectors to attach, and waits for the viewer's credentials: + * sampling against an empty set would list connected providers and then + * reshuffle when the query lands. Build lists personalized suggestions, + * re-sampled whenever signals resolve, and falls back to * {@link INITIAL_ACTIONS} until the credential and service queries have loaded * — and stays there for users with no connections — so first paint never - * flashes. + * flashes. The store's default mode is Build, so the server render never + * shows the sampled Search list. */ - const actions = useMemo(() => { + const buildActions = useMemo(() => { const personalized = services.length > 0 && connectedProviders.size > 0 if (!personalized) return INITIAL_ACTIONS return computeActions(services, signals) }, [connectedProviders, services, signals]) + const actions = buildActions const handleSelect = (action: Action, position: number) => { captureEvent(posthog, 'suggested_action_clicked', { @@ -329,8 +327,8 @@ export function SuggestedActions({ onSelectPrompt }: SuggestedActionsProps) { onSelectPrompt(action.prompt) return } - const match = resolveOAuthServiceForSlug(action.slug) - if (match) setOAuthTarget(match) + const target = resolveOAuthServiceForSlug(action.slug) + if (target) setOAuthTarget(target) } const handleToggleExpanded = () => { @@ -351,7 +349,7 @@ export function SuggestedActions({ onSelectPrompt }: SuggestedActionsProps) { aria-expanded={expanded} className='group/toggle flex w-full cursor-pointer items-center gap-2' > - Suggested actions + {HEADINGS[mode]} {/* * Revealed by hovering anywhere in the section — the group sits on the * section wrapper rather than this row, so the action rows below arm it just @@ -376,28 +374,34 @@ export function SuggestedActions({ onSelectPrompt }: SuggestedActionsProps) { `collapsible-up`/`-down` interpolate height alone, so a margin here would hold its full value through the close and then vanish on unmount, snapping the content below up. */} -
- {actions.map((action, i) => { - const Icon = action.icon - return ( - - ) - })} -
+ {mode !== 'build' && workspaceId ? ( +
+ +
+ ) : ( +
+ {actions.map((action, i) => { + const Icon = action.icon + return ( + + ) + })} +
+ )} {oauthTarget && workspaceId && ( diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/types.ts b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/types.ts new file mode 100644 index 00000000000..9e39d99dbc1 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/types.ts @@ -0,0 +1,20 @@ +import type { ComponentType, CSSProperties } from 'react' + +export type ActionIcon = ComponentType<{ className?: string; style?: CSSProperties }> + +/** What the OAuth connect modal needs to start a connection for one service. */ +export interface OAuthConnectTarget { + providerId: string + requiredScopes: readonly string[] + serviceName: string + serviceIcon: ComponentType<{ className?: string }> +} + +/** + * One suggested-action row. `prompt` rows populate the input with a curated + * prompt; `integration` rows resolve their OAuth service from the catalog slug + * on click and open the OAuth connect modal. + */ +export type Action = + | { kind: 'prompt'; id: string; label: string; icon: ActionIcon; prompt: string } + | { kind: 'integration'; id: string; label: string; icon: ActionIcon; slug: string } diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/weighted-sample.ts b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/weighted-sample.ts new file mode 100644 index 00000000000..b8b6c66607a --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/suggested-actions/weighted-sample.ts @@ -0,0 +1,27 @@ +import { randomFloat } from '@sim/utils/random' + +/** + * Weighted sampling without replacement. Each pick's probability is + * proportional to its weight, so the set stays varied while staying relevant. + * A constant weight yields a uniform sample. + */ +export function weightedSample( + pool: readonly T[], + n: number, + weightOf: (item: T) => number +): T[] { + const remaining = pool.map((item) => ({ item, weight: Math.max(weightOf(item), 0) })) + const out: T[] = [] + while (out.length < n && remaining.length > 0) { + const total = remaining.reduce((sum, entry) => sum + entry.weight, 0) + if (total <= 0) break + let roll = randomFloat() * total + const index = remaining.findIndex((entry) => { + roll -= entry.weight + return roll <= 0 + }) + const [picked] = remaining.splice(index === -1 ? remaining.length - 1 : index, 1) + out.push(picked.item) + } + return out +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/index.ts index 95d472588c1..7d8bdca03af 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/index.ts @@ -24,6 +24,7 @@ export { export { DropOverlay } from './drop-overlay' export { MicButton } from './mic-button' export { MicrophonePermissionHelp } from './microphone-permission-help' +export { ModeSwitcher } from './mode-switcher' export { PlusMenuDropdown } from './plus-menu-dropdown' export type { PromptEditorInstance, diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/index.ts b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/index.ts new file mode 100644 index 00000000000..46800468812 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/index.ts @@ -0,0 +1 @@ +export { ModeSwitcher } from './mode-switcher' diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx new file mode 100644 index 00000000000..92f6aa38b0e --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.test.tsx @@ -0,0 +1,154 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockCaptureEvent, mockSetSearchQuery, mockSetSearchFilters, modeState } = vi.hoisted( + () => ({ + mockCaptureEvent: vi.fn(), + mockSetSearchQuery: vi.fn(), + mockSetSearchFilters: vi.fn(), + /** The URL `mode` param as the nuqs mock serves it; `set` is the live setter once mounted. */ + modeState: { initial: 'build', set: (_next: string) => {} }, + }) +) + +vi.mock('next/navigation', () => ({ + useParams: () => ({ workspaceId: 'workspace-1' }), +})) +vi.mock('nuqs', async () => { + const { useState } = await import('react') + return { + useQueryState: (key: string) => { + const [mode, setMode] = useState(modeState.initial) + if (key !== 'mode') return [null, mockSetSearchQuery] + modeState.set = setMode + return [mode, setMode] + }, + useQueryStates: () => [{}, mockSetSearchFilters], + } +}) +vi.mock('posthog-js/react', () => ({ usePostHog: () => null })) +vi.mock('@/lib/posthog/client', () => ({ captureEvent: mockCaptureEvent })) + +import { ModeSwitcher } from '@/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher' + +let root: Root | null = null +let container: HTMLDivElement | null = null + +function mount() { + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) + act(() => root?.render()) +} + +function trigger(): HTMLButtonElement { + const node = container?.querySelector('button') + if (!node) throw new Error('Switcher trigger did not render') + return node +} + +/** Opens the menu the way a pointer does — Radix opens on `pointerdown`. */ +function openMenu() { + act(() => { + trigger().dispatchEvent(new MouseEvent('pointerdown', { bubbles: true, button: 0 })) + }) +} + +function items(): HTMLElement[] { + return Array.from(document.querySelectorAll('[role="menuitem"]')) +} + +function select(index: number) { + act(() => { + items()[index].dispatchEvent(new MouseEvent('click', { bubbles: true, button: 0 })) + }) +} + +beforeEach(() => { + mockCaptureEvent.mockClear() + mockSetSearchQuery.mockClear() + mockSetSearchFilters.mockClear() + modeState.initial = 'build' +}) + +afterEach(() => { + if (root) act(() => root?.unmount()) + container?.remove() + root = null + container = null +}) + +describe('ModeSwitcher', () => { + it('renders the active mode as a label-only round chip and defaults to Build', () => { + mount() + + const button = trigger() + expect(button.textContent).toBe('Build') + expect(button.getAttribute('aria-label')).toBe('Mode: Build') + expect(button.className).toContain('h-[30px]') + expect(button.className).toContain('rounded-full') + expect(button.className).not.toContain('rounded-lg') + expect(button.className).toContain('hover-hover:bg-[var(--surface-hover)]') + expect(button.querySelector('svg')).toBeNull() + }) + + it('lists every mode and checks the active one', () => { + mount() + openMenu() + + const rows = items() + expect(rows.map((row) => row.textContent)).toEqual(['Build', 'Search', 'Assistant']) + expect(rows[0].querySelector('svg')).not.toBeNull() + expect(rows[1].querySelector('svg')).toBeNull() + expect(rows[2].querySelector('svg')).toBeNull() + }) + + it('writes the chosen mode to the URL and reports the change', () => { + mount() + openMenu() + select(1) + + expect(trigger().textContent).toBe('Search') + expect(mockCaptureEvent).toHaveBeenCalledWith(null, 'chat_mode_changed', { + workspace_id: 'workspace-1', + mode: 'search', + }) + expect(mockSetSearchQuery).not.toHaveBeenCalled() + }) + + it('reads the mode from the URL on mount', () => { + modeState.initial = 'assistant' + mount() + + expect(trigger().textContent).toBe('Assistant') + expect(trigger().getAttribute('aria-label')).toBe('Mode: Assistant') + }) + + it('drops the search query from the URL when leaving Search', () => { + modeState.initial = 'search' + mount() + openMenu() + select(0) + + expect(trigger().textContent).toBe('Build') + expect(mockSetSearchQuery).toHaveBeenCalledWith(null, { history: 'replace', scroll: false }) + expect(mockSetSearchFilters).toHaveBeenCalledWith( + { source: null, updated: null }, + { history: 'replace', scroll: false } + ) + }) + + it('does not report re-selecting the active mode', () => { + mount() + openMenu() + select(0) + + expect(trigger().textContent).toBe('Build') + expect(mockCaptureEvent).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx new file mode 100644 index 00000000000..e7ec9a3b6a8 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/components/mode-switcher/mode-switcher.tsx @@ -0,0 +1,76 @@ +'use client' + +import { memo } from 'react' +import { + Chip, + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuItemLabel, + DropdownMenuTrigger, +} from '@sim/emcn' +import { Check } from '@sim/emcn/icons' +import { useParams } from 'next/navigation' +import { useQueryState, useQueryStates } from 'nuqs' +import { usePostHog } from 'posthog-js/react' +import { captureEvent } from '@/lib/posthog/client' +import { useMothershipMode } from '@/app/workspace/[workspaceId]/home/hooks/use-mothership-mode' +import { + CLEARED_SEARCH_FILTERS, + MOTHERSHIP_MODES, + type MothershipMode, + resourceUrlKeys, + searchFilterParsers, + searchQueryParam, +} from '@/app/workspace/[workspaceId]/home/search-params' + +const MODE_LABELS: Record = { + build: 'Build', + search: 'Search', + assistant: 'Assistant', +} + +/** + * The composer's Build / Search / Assistant switcher: a label-only `Chip` in its `round` + * shape — chip chrome throughout (`--text-body` label, `--surface-hover` on + * hover, no text-color shift), fully round to sit in the toolbar's row of + * round controls — opening a menu that checks the active mode, as + * `ChipDropdown` does. + */ +export const ModeSwitcher = memo(function ModeSwitcher() { + const { workspaceId } = useParams<{ workspaceId: string }>() + const posthog = usePostHog() + const [mode, setMode] = useMothershipMode() + + const [, setSearchQueryParam] = useQueryState(searchQueryParam.key, searchQueryParam.parser) + const [, setSearchFilters] = useQueryStates(searchFilterParsers, resourceUrlKeys) + + /** Leaving Search drops the query from the URL, so a clean URL always means no search is showing. */ + const handleSelect = (next: MothershipMode) => { + if (next === mode) return + void setMode(next) + if (next !== 'search') { + void setSearchQueryParam(null, { history: 'replace', scroll: false }) + void setSearchFilters(CLEARED_SEARCH_FILTERS, { history: 'replace', scroll: false }) + } + captureEvent(posthog, 'chat_mode_changed', { workspace_id: workspaceId, mode: next }) + } + + return ( + + + + {MODE_LABELS[mode]} + + + + {MOTHERSHIP_MODES.map((option) => ( + handleSelect(option)}> + + {option === mode && } + + ))} + + + ) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx index 13c23d419cf..5b1be141001 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/components/user-input/user-input.tsx @@ -25,6 +25,7 @@ import { DropOverlay, MicButton, MicrophonePermissionHelp, + ModeSwitcher, PromptEditor, SendButton, usePromptEditor, @@ -69,6 +70,21 @@ interface UserInputProps { onStopGeneration: () => void isInitialView?: boolean onSendQueuedHead?: () => void + /** + * Whether the composer offers Search mode. Only the Home composer answers a + * search with documents; the workflow copilot always talks to the agent, so + * it must not show a mode it cannot honour. + */ + canSearch?: boolean + /** + * Whether the text is cleared once submitted. A search keeps its query in + * the box, the way a search bar does, so it can be read and refined against + * the results; a message to the agent clears, since it now lives in the + * transcript. Defaults to clearing. + */ + clearOnSubmit?: boolean + /** Called when the text becomes empty after having had content, such as a search being cleared. */ + onCleared?: () => void onEditQueuedTail?: () => void } @@ -80,6 +96,8 @@ export interface UserInputHandle { * names chip with brand icons. Focuses the input and places the caret at the * end. Does NOT submit. Safe to call with the same text twice in a row. */ populatePrompt: (text: string) => void + /** Empties the composer and its draft, as a send does; for a question handed to the agent from outside the box. */ + clear: () => void } /** @@ -97,6 +115,9 @@ const UserInputImpl = forwardRef(function UserI isInitialView = true, onSendQueuedHead, onEditQueuedTail, + canSearch = false, + clearOnSubmit = true, + onCleared, }, ref ) { @@ -157,6 +178,8 @@ const UserInputImpl = forwardRef(function UserI const draftScopeKeyRef = useRef(draftScopeKey) draftScopeKeyRef.current = draftScopeKey + const clearOnSubmitRef = useRef(clearOnSubmit) + clearOnSubmitRef.current = clearOnSubmit const hasRestoredDraftRef = useRef(false) useEffect(() => { @@ -202,6 +225,15 @@ const UserInputImpl = forwardRef(function UserI } }, []) // eslint-disable-line react-hooks/exhaustive-deps -- intentional mount-only restore + const onClearedRef = useRef(onCleared) + onClearedRef.current = onCleared + const hadTextRef = useRef(false) + useEffect(() => { + const hasText = editor.value.trim().length > 0 + if (hadTextRef.current && !hasText) onClearedRef.current?.() + hadTextRef.current = hasText + }, [editor.value]) + const isFirstSaveRef = useRef(true) const draftSaveTimerRef = useRef(null) const pendingDraftRef = useRef<{ key: string; payload: DraftPayload } | null>(null) @@ -404,6 +436,7 @@ const UserInputImpl = forwardRef(function UserI currentEditor.setContexts(msg.contexts ?? []) currentEditor.focusAtEnd() }, + clear: clearComposer, populatePrompt: (text: string) => { // `text` is a curated prompt, so opt its bare integration names into // `@`-mention form before chipification (the auto-mention pipeline only @@ -512,11 +545,33 @@ const UserInputImpl = forwardRef(function UserI return () => window.cancelAnimationFrame(raf) }, [textareaRef]) + /** + * Menu rows are excluded alongside buttons: the mode switcher's items are + * portaled, so their clicks still bubble here through the React tree. + */ const handleContainerClick = (e: React.MouseEvent) => { - if ((e.target as HTMLElement).closest('button, [role="dialog"]')) return + if ((e.target as HTMLElement).closest('button, [role="dialog"], [role="menu"]')) return textareaRef.current?.focus() } + /** Empties the text, chips, attachments, transcript, and the saved draft in one step. */ + const clearComposer = useCallback(() => { + editorRef.current.clear() + sttPrefixRef.current = '' + if (draftSaveTimerRef.current !== null) { + window.clearTimeout(draftSaveTimerRef.current) + draftSaveTimerRef.current = null + } + pendingDraftRef.current = null + if (draftScopeKeyRef.current) { + useMothershipDraftsStore.getState().clearDraft(draftScopeKeyRef.current) + } + /** The chips are gone with the text, and clearing is not a removal to report. */ + prevSelectedContextsRef.current = [] + resetTranscript() + filesRef.current.clearAttachedFiles() + }, [resetTranscript]) + const handleSubmit = useCallback(() => { const currentFiles = filesRef.current const currentEditor = editorRef.current @@ -541,20 +596,13 @@ const UserInputImpl = forwardRef(function UserI fileAttachmentsForApi.length > 0 ? fileAttachmentsForApi : undefined, activeContexts.length > 0 ? activeContexts : undefined ) - currentEditor.clear() - sttPrefixRef.current = '' - if (draftSaveTimerRef.current !== null) { - window.clearTimeout(draftSaveTimerRef.current) - draftSaveTimerRef.current = null - } - pendingDraftRef.current = null - if (draftScopeKeyRef.current) { - useMothershipDraftsStore.getState().clearDraft(draftScopeKeyRef.current) - } - resetTranscript() - currentFiles.clearAttachedFiles() - prevSelectedContextsRef.current = [] - }, [onSubmit, resetTranscript]) + /** + * A composer that keeps its text (Search mode) keeps its attachments and + * chips too: the search took the query alone, and the person may hand the + * rest to the agent next. + */ + if (clearOnSubmitRef.current) clearComposer() + }, [onSubmit, clearComposer]) /** * Enter policy for the editor: mirror canSubmit's uploading guard (Enter @@ -678,6 +726,7 @@ const UserInputImpl = forwardRef(function UserI
+ {canSearch && } {isSttSupported && ( { + void setSearchQueryParam(query || null) + void setSearchFilters(CLEARED_SEARCH_FILTERS) + }, + [setSearchQueryParam, setSearchFilters] + ) + const [composerMode, setComposerMode] = useMothershipMode() + /** + * A link that carries a query but no mode opens in Search with the query in + * the box; the composer follows the live query the same way (below), so the + * box and the results never show two different queries. + */ + useEffect(() => { + if (searchQuery.trim() && composerMode === 'build') void setComposerMode('search') + }, [searchQuery, composerMode, setComposerMode]) const hasCheckedLandingStorageRef = useRef(false) const initialViewInputRef = useRef(null) const initialViewUserInputRef = useRef(null) + const chatViewUserInputRef = useRef(null) const [isInputEntering, setIsInputEntering] = useState(false) @@ -422,7 +470,12 @@ export function Home({ chatId, userName, userId }: HomeProps) { }, [workspaceId, getCurrentRequestId, stopGeneration]) const handleSubmit = useCallback( - (text: string, fileAttachments?: FileAttachmentForApi[], contexts?: ChatContext[]) => { + async ( + text: string, + fileAttachments?: FileAttachmentForApi[], + contexts?: ChatContext[], + modeOverride?: MothershipMode + ) => { const trimmed = text.trim() if (!trimmed && !(fileAttachments && fileAttachments.length > 0)) return @@ -433,16 +486,104 @@ export function Home({ chatId, userName, userId }: HomeProps) { is_new_task: !chatId, }) + /** + * Search lists documents, not a turn of the agent, and only a query can + * be searched: attachments alone have nothing to search for. Assistant + * makes the query a turn of the agent grounded in the sources. + */ + const mode = modeOverride ?? composerMode + const answering = mode === 'assistant' + if (mode === 'search') { + /** A search sends nothing, so an edit in progress is released rather than left waiting. */ + if (editingQueuedId) cancelQueueEdit() + if (trimmed) setSearchQuery(trimmed) + return + } + if (initialViewInputRef.current) { setIsInputEntering(true) } prepareResourceViewForAgentTurn() - sendMessage(trimmed || 'Analyze the attached file(s).', fileAttachments, contexts) + /** + * An Assistant turn is grounded in the searched bases, read from the + * query cache the Search panel shares: instant once loaded, and awaited + * the one time a question is typed before the list has arrived. + */ + const turnContexts = answering + ? withSearchedKnowledgeContexts( + contexts, + searchedKnowledgeBases( + await queryClient.ensureQueryData({ + queryKey: knowledgeKeys.list(workspaceId, 'active'), + queryFn: ({ signal }) => fetchKnowledgeBases(workspaceId, 'active', signal), + staleTime: KNOWLEDGE_BASE_LIST_STALE_TIME, + }), + workspaceId + ) + ) + : contexts + sendMessage( + trimmed || 'Analyze the attached file(s).', + fileAttachments, + turnContexts, + answering ? { requestMode: 'ask' } : undefined + ) + }, + [ + workspaceId, + chatId, + composerMode, + editingQueuedId, + cancelQueueEdit, + prepareResourceViewForAgentTurn, + queryClient, + sendMessage, + setSearchQuery, + ] + ) + + /** + * A queued message re-enters the composer in the mode it was written in: an + * Assistant question edits as an Assistant question, and never as a Search, + * which submits nothing and would leave the edit stranded. + */ + const restoreQueuedMode = useCallback( + (requestMode: QueuedMessage['requestMode']) => { + void setComposerMode(requestMode === 'ask' ? 'assistant' : 'build') }, - [workspaceId, chatId, prepareResourceViewForAgentTurn, sendMessage] + [setComposerMode] ) + /** An emptied search box returns to the sources; a send in any other mode has no search to clear. */ + const clearSearch = useCallback(() => { + if (searchQueryValue !== null) setSearchQuery('') + }, [searchQueryValue, setSearchQuery]) + + /** + * Summarize or Answer on a result: switch to Assistant and hand the question + * to it. The submit reads the mode from this render, so it is sent as an + * Assistant turn directly rather than waiting for the URL to update, and the + * box is emptied as a send empties it, so the query does not linger as a + * draft under the answer. + */ + const handleSummarize = (prompt: string) => { + void setComposerMode('assistant') + setSearchQuery('') + initialViewUserInputRef.current?.clear() + chatViewUserInputRef.current?.clear() + void handleSubmit(prompt, undefined, undefined, 'assistant') + } + const showSearchResults = composerMode === 'search' && searchQuery.trim().length > 0 + const searchResults = showSearchResults ? ( + + ) : null + /** * Handles cross-surface send requests (terminal/console "Fix in Chat", the * log "Troubleshoot in Chat" action). `preventDefault` claims the event so a @@ -457,6 +598,7 @@ export function Home({ chatId, userName, userId }: HomeProps) { prepareResourceViewForAgentTurn() sendMessage(detail.message, detail.fileAttachments, detail.contexts, { ...(detail.resumeUserMessageId ? { resumeUserMessageId: detail.resumeUserMessageId } : {}), + ...(detail.requestMode ? { requestMode: detail.requestMode } : {}), }) } window.addEventListener(MOTHERSHIP_SEND_MESSAGE_EVENT, handler) @@ -491,6 +633,7 @@ export function Home({ chatId, userName, userId }: HomeProps) { ...(handoff.resumeUserMessageId ? { resumeUserMessageId: handoff.resumeUserMessageId } : {}), + ...(handoff.requestMode ? { requestMode: handoff.requestMode } : {}), }) return } @@ -658,20 +801,25 @@ export function Home({ chatId, userName, userId }: HomeProps) { > {/* Anchored out of flow so expanding/collapsing never shifts the centered input */}
- - initialViewUserInputRef.current?.populatePrompt(prompt) - } - /> + {searchResults ?? ( + + initialViewUserInputRef.current?.populatePrompt(prompt) + } + /> + )}
@@ -681,9 +829,16 @@ export function Home({ chatId, userName, userId }: HomeProps) { workspaceId={workspaceId} messages={messages} isSending={isSending} + searchResults={searchResults} + searchQuery={searchQuery} + userInputRef={chatViewUserInputRef} + onRestoreQueuedMode={restoreQueuedMode} isReconnecting={isReconnecting} isLoading={showChatSkeleton} onSubmit={handleSubmit} + canSearch + clearOnSubmit={composerMode !== 'search'} + onCleared={clearSearch} onStopGeneration={handleStopGeneration} messageQueue={messageQueue} editingQueuedId={editingQueuedId} diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/chat-url.test.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/chat-url.test.ts new file mode 100644 index 00000000000..c2a8e485d3f --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/chat-url.test.ts @@ -0,0 +1,28 @@ +/** + * @vitest-environment jsdom + */ +import { describe, expect, it } from 'vitest' +import { chatUrl } from '@/app/workspace/[workspaceId]/home/hooks/chat-url' + +function withSearch(search: string) { + window.history.replaceState(null, '', `/workspace/ws-1/home${search}`) +} + +describe('chatUrl', () => { + it('carries the mode and the open resource onto the chat path', () => { + withSearch('?mode=assistant&resource=res-1') + expect(chatUrl('ws-1', 'chat-1')).toBe( + '/workspace/ws-1/chat/chat-1?mode=assistant&resource=res-1' + ) + }) + + it('leaves a search query and its filters behind', () => { + withSearch('?q=volvo&source=gmail&updated=7d&mode=assistant') + expect(chatUrl('ws-1', 'chat-1')).toBe('/workspace/ws-1/chat/chat-1?mode=assistant') + }) + + it('produces a clean path when nothing belongs on the chat', () => { + withSearch('?q=volvo') + expect(chatUrl('ws-1', 'chat-1')).toBe('/workspace/ws-1/chat/chat-1') + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/chat-url.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/chat-url.ts new file mode 100644 index 00000000000..2046927bc57 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/chat-url.ts @@ -0,0 +1,22 @@ +import { modeParam, resourceParam } from '@/app/workspace/[workspaceId]/home/search-params' + +/** The composer's URL state that belongs on a chat page: the mode and the open resource. */ +const CHAT_URL_PARAMS = [modeParam.key, resourceParam.key] as const + +/** + * The URL a new chat is handed off to once the server names it. Only the + * params that belong on a chat ride along, so the mode survives the path swap + * (the first Assistant message must not bounce the person back to Build) while + * a search's `q` and filters, which never join a transcript, are left behind + * whatever the URL held at that instant. + */ +export function chatUrl(workspaceId: string, chatId: string): string { + const current = new URLSearchParams(window.location.search) + const carried = new URLSearchParams() + for (const key of CHAT_URL_PARAMS) { + const value = current.get(key) + if (value) carried.set(key, value) + } + const search = carried.toString() + return `/workspace/${workspaceId}/chat/${chatId}${search ? `?${search}` : ''}` +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/index.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/index.ts index 8c1fa13edd3..e6afef301d8 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/index.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/index.ts @@ -5,4 +5,5 @@ export { shouldActivateResourceEvent, useChat, } from './use-chat' +export { useMothershipMode } from './use-mothership-mode' export { useMothershipResize } from './use-mothership-resize' diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-session-event.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-session-event.ts index 6db80d081c7..a4e077be89d 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-session-event.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/stream/handle-session-event.ts @@ -1,6 +1,7 @@ import { getLiveAssistantMessageId } from '@/lib/copilot/chat/effective-transcript' import { MothershipStreamV1SessionKind } from '@/lib/copilot/generated/mothership-stream-v1' import type { PersistedStreamEventEnvelope } from '@/lib/copilot/request/session/contract' +import { chatUrl } from '@/app/workspace/[workspaceId]/home/hooks/chat-url' import type { StreamLoopContext } from '@/app/workspace/[workspaceId]/home/hooks/stream/stream-context' import { type MothershipChatHistory, mothershipChatKeys } from '@/hooks/queries/mothership-chats' @@ -56,11 +57,7 @@ export function handleSessionEvent(ctx: StreamLoopContext, parsed: SessionEvent) } deps.setPendingMessages([]) if (!deps.workflowIdRef.current) { - window.history.replaceState( - null, - '', - `/workspace/${deps.workspaceId}/chat/${payloadChatId}` - ) + window.history.replaceState(null, '', chatUrl(deps.workspaceId, payloadChatId)) } } } diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts index 6114e8057bd..a9edcc80d82 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts @@ -98,6 +98,7 @@ import { import { sendMothershipMessage } from '@/lib/mothership/events' import { initTerminalTransport } from '@/lib/terminal/transport' import { getQueryClient } from '@/app/_shell/providers/get-query-client' +import { chatUrl } from '@/app/workspace/[workspaceId]/home/hooks/chat-url' import { useFilePreviewController } from '@/app/workspace/[workspaceId]/home/hooks/preview' import { captureResourceActivityScope, @@ -142,6 +143,7 @@ import type { WorkflowMetadata } from '@/stores/workflows/registry/types' import type { ChatMessage, ChatMessageContext, + ChatRequestMode, ContentBlock, FileAttachmentForApi, GenericResourceData, @@ -158,6 +160,8 @@ export interface SendMessageOptions { * attempts instead of opening a second chat. */ resumeUserMessageId?: string + /** Asked for beyond the default agent turn; `ask` answers from the attached knowledge alone. */ + requestMode?: ChatRequestMode } /** @@ -181,6 +185,7 @@ interface StartSendMessageOptions { * opening a second chat and billing a second turn. */ resumeUserMessageId?: string + requestMode?: ChatRequestMode } /** A send an unmount cleanup withdrew, as handed to the next chat surface. */ @@ -189,6 +194,7 @@ interface WithdrawnSend { fileAttachments?: FileAttachmentForApi[] contexts?: ChatContext[] userMessageId: string + requestMode?: ChatRequestMode } export interface UseChatReturn { @@ -299,6 +305,7 @@ interface QueuedSendHandoffState { message: string fileAttachments?: FileAttachmentForApi[] contexts?: ChatContext[] + requestMode?: ChatRequestMode requestedAt: number resolveAttempts?: number } @@ -1779,7 +1786,7 @@ export function useChat( !workflowIdRef.current && typeof window !== 'undefined' ) { - window.history.replaceState(null, '', `/workspace/${workspaceId}/chat/${chatId}`) + window.history.replaceState(null, '', chatUrl(workspaceId, chatId)) } if (options?.invalidateList) { queryClient.invalidateQueries({ queryKey: mothershipChatKeys.list(workspaceId) }) @@ -3698,7 +3705,8 @@ export function useChat( message: string, fileAttachments?: FileAttachmentForApi[], contexts?: ChatContext[], - resumeUserMessageId?: string + resumeUserMessageId?: string, + requestMode?: ChatRequestMode ): QueuedMothershipMessage => { const id = generateId() const handoffChatId = selectedChatIdRef.current ?? chatIdRef.current @@ -3719,6 +3727,7 @@ export function useChat( fileAttachments, contexts, ...(resumeUserMessageId ? { resumeUserMessageId } : {}), + ...(requestMode ? { requestMode } : {}), ...(supersededStreamId || handoffChatId ? { queuedSendHandoff: { @@ -3860,6 +3869,7 @@ export function useChat( message, ...(fileAttachments ? { fileAttachments } : {}), ...(contexts ? { contexts } : {}), + ...(options?.requestMode ? { requestMode: options.requestMode } : {}), requestedAt: Date.now(), }) } @@ -4086,6 +4096,7 @@ export function useChat( ...(fileAttachments && fileAttachments.length > 0 ? { fileAttachments } : {}), ...(resourceAttachments ? { resourceAttachments } : {}), ...(contexts && contexts.length > 0 ? { contexts } : {}), + ...(options?.requestMode ? { mode: options.requestMode } : {}), ...(workflowIdRef.current ? { workflowId: workflowIdRef.current } : {}), // Desktop-only capabilities (local filesystem tools, browser // subagent) — the server gates the features on these flags. @@ -4286,7 +4297,13 @@ export function useChat( const handOffWithdrawnSend = useCallback( (send: WithdrawnSend) => { if ( - sendMothershipMessage(send.content, send.contexts, send.fileAttachments, send.userMessageId) + sendMothershipMessage( + send.content, + send.contexts, + send.fileAttachments, + send.userMessageId, + send.requestMode + ) ) { return } @@ -4296,6 +4313,7 @@ export function useChat( ...(send.contexts?.length ? { contexts: send.contexts } : {}), ...(send.fileAttachments?.length ? { fileAttachments: send.fileAttachments } : {}), resumeUserMessageId: send.userMessageId, + ...(send.requestMode ? { requestMode: send.requestMode } : {}), }, workspaceId ) @@ -4325,6 +4343,7 @@ export function useChat( content: message, fileAttachments, contexts, + requestMode: options?.requestMode, }) queueStore.setEditing(activeChatKey, null) // Resume dispatch if it paused on this slot. @@ -4352,7 +4371,13 @@ export function useChat( ) { queueStore.enqueue( activeChatKey, - createQueuedMessage(message, fileAttachments, contexts, options?.resumeUserMessageId) + createQueuedMessage( + message, + fileAttachments, + contexts, + options?.resumeUserMessageId, + options?.requestMode + ) ) if (pendingStopPromiseRef.current || (queuedAheadCount > 0 && !sendingRef.current)) { void enqueueQueueDispatchRef.current({ type: 'send_head' }) @@ -4373,6 +4398,7 @@ export function useChat( fileAttachments, contexts, userMessageId: result.userMessageId, + ...(options?.requestMode ? { requestMode: options.requestMode } : {}), } if (activeChatKey.startsWith(PENDING_CHAT_KEY_PREFIX)) { handOffWithdrawnSend(withdrawn) @@ -4382,7 +4408,13 @@ export function useChat( .getState() .enqueue( activeChatKey, - createQueuedMessage(message, fileAttachments, contexts, result.userMessageId) + createQueuedMessage( + message, + fileAttachments, + contexts, + result.userMessageId, + options?.requestMode + ) ) }, [workspaceId, createQueuedMessage, startSendMessage, handOffWithdrawnSend] @@ -4556,6 +4588,7 @@ export function useChat( recoveringQueuedSendHandoffRef.current = { id: handoff.id, ownerId: claimOwnerId } void startSendMessage(handoff.message, handoff.fileAttachments, handoff.contexts, { pendingStop: null, + ...(handoff.requestMode ? { requestMode: handoff.requestMode } : {}), queuedSendHandoff: { id: handoff.id, chatId: handoff.chatId, @@ -4978,6 +5011,7 @@ export function useChat( content: dispatched.content, fileAttachments: dispatched.fileAttachments, contexts: dispatched.contexts, + ...(dispatched.requestMode ? { requestMode: dispatched.requestMode } : {}), userMessageId: withdrawnUserMessageId, }) return @@ -5016,6 +5050,7 @@ export function useChat( ...(liveMsg.resumeUserMessageId ? { resumeUserMessageId: liveMsg.resumeUserMessageId } : {}), + ...(liveMsg.requestMode ? { requestMode: liveMsg.requestMode } : {}), } ) diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/use-mothership-mode.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/use-mothership-mode.ts new file mode 100644 index 00000000000..99d8c5ea2d0 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/use-mothership-mode.ts @@ -0,0 +1,13 @@ +'use client' + +import { useQueryState } from 'nuqs' +import { modeParam } from '@/app/workspace/[workspaceId]/home/search-params' + +/** + * The composer's mode, read from and written to the URL's `mode` param so a + * refresh, back, forward, or shared link lands in the same mode, as Glean's + * separate Search and Assistant routes do. Build is the clean URL. + */ +export function useMothershipMode() { + return useQueryState(modeParam.key, modeParam.parser) +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/search-params.ts b/apps/sim/app/workspace/[workspaceId]/home/search-params.ts index ef850466f69..101ec04207e 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/search-params.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/search-params.ts @@ -1,4 +1,4 @@ -import { parseAsString } from 'nuqs/server' +import { parseAsString, parseAsStringLiteral } from 'nuqs/server' /** * Co-located, typed URL query-param definition for the home/Chat surface. @@ -25,3 +25,54 @@ export const resourceUrlKeys = { history: 'replace', clearOnDefault: true, } as const + +/** + * `q` is the composer's Search-mode query, so a search is a shareable, + * bookmarkable link. Present only while a search is showing: it is dropped + * when the box empties, on Summarize, and when the mode leaves Search. The + * composer reads it once on mount to restore the query and the Search mode. + * Filter-like, so it replaces the history entry. + */ +export const searchQueryParam = { + key: 'q', + parser: parseAsString, +} as const + +/** The composer's modes: the agent, enterprise search, or the assistant answering from the sources. */ +export const MOTHERSHIP_MODES = ['build', 'search', 'assistant'] as const + +export type MothershipMode = (typeof MOTHERSHIP_MODES)[number] + +/** + * `mode` is the composer's mode, so a refresh, back, forward, or shared link + * lands in the same mode, as Glean's separate Search and Assistant routes do. + * Build is the default and the clean URL. A view change rather than a + * destination, so it replaces the history entry. + */ +export const modeParam = { + key: 'mode', + parser: parseAsStringLiteral(MOTHERSHIP_MODES) + .withDefault('build') + .withOptions({ history: 'replace', clearOnDefault: true }), +} as const + +/** The recency windows a search can be narrowed to. */ +export const UPDATED_WINDOWS = [ + { id: 'any', label: 'Any time', days: null }, + { id: '7d', label: 'Past week', days: 7 }, + { id: '30d', label: 'Past month', days: 30 }, +] as const +const UPDATED_WINDOW_IDS = UPDATED_WINDOWS.map((window) => window.id) + +/** + * The result filters, beside `q`, so a narrowed search is the same shareable + * link as the search itself. `source` is a connector type or `upload`, absent + * for every source; both are dropped with the query. + */ +export const searchFilterParsers = { + source: parseAsString, + updated: parseAsStringLiteral(UPDATED_WINDOW_IDS).withDefault('any'), +} as const + +/** Every search param at its default: what leaving a search writes. */ +export const CLEARED_SEARCH_FILTERS = { source: null, updated: null } as const diff --git a/apps/sim/app/workspace/[workspaceId]/home/types.ts b/apps/sim/app/workspace/[workspaceId]/home/types.ts index 983cc4eba67..778f6f5ba68 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/types.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/types.ts @@ -23,11 +23,19 @@ export interface FileAttachmentForApi { path?: string } +/** + * A request mode a send asks the agent for beyond the default. `ask` is an + * Assistant turn: an answer drawn from the attached knowledge bases first, + * with a connected integration reached only when those cannot answer. + */ +export type ChatRequestMode = 'ask' + export interface QueuedMessage { id: string content: string fileAttachments?: FileAttachmentForApi[] contexts?: ChatContext[] + requestMode?: ChatRequestMode } export const ToolCallStatus = { diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx index faa56a106b9..7e0fdeeb337 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx @@ -30,12 +30,24 @@ import { type OAuthProvider, } from '@/lib/oauth' import { ConnectOAuthModal } from '@/app/workspace/[workspaceId]/components/connect-oauth-modal' +import { + ConnectorAccessField, + type ConnectorAccessSelection, +} from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field' import { ConnectorConfigFields } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connector-config-fields' import { hasWorkspaceMaxConnectorAccess } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connector-entitlements' -import { SYNC_INTERVALS } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/consts' +import { + BROWSE_WITH_HINT, + SYNC_INTERVALS, +} from '@/app/workspace/[workspaceId]/knowledge/[id]/components/consts' import { MaxBadge } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/max-badge' import { useConnectorConfigFields } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields' +import { + memberCapFieldIds, + useConnectorMemberGroupOptions, +} from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-member-group-options' import { useWorkspaceHostContext } from '@/app/workspace/[workspaceId]/providers/workspace-host-provider' +import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' import { getBlock } from '@/blocks' import { withBrandIcon } from '@/blocks/brand-icon' import { getTileIconColorClass } from '@/blocks/icon-color' @@ -47,6 +59,8 @@ import { useCredentialRefreshTriggers } from '@/hooks/use-credential-refresh-tri const CONNECTOR_ENTRIES = Object.entries(CONNECTOR_META_REGISTRY) +const WORKSPACE_ACCESS: ConnectorAccessSelection = { accessMode: 'workspace' } + interface AddConnectorModalProps { open: boolean onOpenChange: (open: boolean) => void @@ -68,6 +82,7 @@ export function AddConnectorModal({ const [selectedType, setSelectedType] = useState(initialConnectorType ?? null) const [syncInterval, setSyncInterval] = useState(1440) const [selectedCredentialId, setSelectedCredentialId] = useState(null) + const [access, setAccess] = useState(WORKSPACE_ACCESS) const [disabledTagIds, setDisabledTagIds] = useState>(() => new Set()) const [error, setError] = useState(null) const [showOAuthModal, setShowOAuthModal] = useState(false) @@ -77,13 +92,28 @@ export function AddConnectorModal({ const [searchTerm, setSearchTerm] = useState('') const { workspaceId } = useParams<{ workspaceId: string }>() - const { ownerBilling } = useWorkspaceHostContext() + const { ownerBilling, features } = useWorkspaceHostContext() + const { canAdmin } = useUserPermissionsContext() + const memberAccessAvailable = features?.knowledgeMemberAccess === true const { mutate: createConnector, isPending: isCreating } = useCreateConnector() const hasMaxAccess = hasWorkspaceMaxConnectorAccess(ownerBilling) const connectorConfig = selectedType ? CONNECTOR_META_REGISTRY[selectedType] : null const isApiKeyMode = connectorConfig?.auth.mode === 'apiKey' + const isMembersMode = access.accessMode === 'members' + const groupOptions = useConnectorMemberGroupOptions({ + workspaceId, + connectorConfig, + enabled: canAdmin && memberAccessAvailable, + }) + /** Several groups collect this provider's accounts: the admin has to say which. */ + const membersChoiceOpen = + isMembersMode && groupOptions.needsChoice && !access.credentialGroupOptionId + const hiddenCapFieldIds = useMemo( + () => memberCapFieldIds(connectorConfig, access.accessMode), + [connectorConfig, access.accessMode] + ) /** True when the connector declares its key optional (public sources need none). */ const isApiKeyOptional = connectorConfig?.auth.mode === 'apiKey' && connectorConfig.auth.optional === true @@ -140,6 +170,7 @@ export function AddConnectorModal({ setSelectedType(type) setSourceConfig({}) setSelectedCredentialId(null) + setAccess(WORKSPACE_ACCESS) setApiKeyValue('') setApiKeyFocused(false) setDisabledTagIds(new Set()) @@ -166,6 +197,8 @@ export function AddConnectorModal({ if (!connectorConfig) return false if (isApiKeyMode) { if (!isApiKeyOptional && !apiKeyValue.trim()) return false + } else if (isMembersMode) { + if (membersChoiceOpen) return false } else { if (!effectiveCredentialId) return false } @@ -173,12 +206,16 @@ export function AddConnectorModal({ for (const field of connectorConfig.configFields) { if (!field.required) continue if (!isFieldVisible(field)) continue + if (hiddenCapFieldIds.has(field.id)) continue if (!isFieldPopulated(field)) return false } return true }, [ connectorConfig, isApiKeyMode, + isMembersMode, + membersChoiceOpen, + hiddenCapFieldIds, isApiKeyOptional, apiKeyValue, effectiveCredentialId, @@ -193,6 +230,7 @@ export function AddConnectorModal({ const resolvedConfig: Record = {} for (const [key, value] of Object.entries(resolveSourceConfig())) { + if (hiddenCapFieldIds.has(key)) continue if (Array.isArray(value)) { if (value.length > 0) resolvedConfig[key] = value } else if (typeof value === 'string') { @@ -217,7 +255,13 @@ export function AddConnectorModal({ ? apiKeyValue.trim() ? { apiKey: apiKeyValue } : {} - : { credentialId: effectiveCredentialId! }), + : isMembersMode + ? { + accessMode: 'members' as const, + credentialGroupId: access.credentialGroupId, + credentialGroupOptionId: access.credentialGroupOptionId, + } + : { credentialId: effectiveCredentialId! }), sourceConfig: finalSourceConfig, syncIntervalMinutes: syncInterval, }, @@ -303,6 +347,17 @@ export function AddConnectorModal({ ) : connectorConfig ? ( <> + {!isApiKeyMode && memberAccessAvailable && ( + + )} + {isApiKeyMode ? ( ) : ( - + + isFieldVisible(field) && !hiddenCapFieldIds.has(field.id) + } onFieldChange={handleFieldChange} onToggleCanonicalMode={toggleCanonicalMode} disabled={isCreating} @@ -440,7 +501,13 @@ export function AddConnectorModal({ onOpenChange(false)} primaryAction={{ - label: isCreating ? 'Connecting…' : 'Connect & Sync', + label: isCreating + ? isMembersMode + ? 'Creating…' + : 'Connecting…' + : isMembersMode + ? 'Create & Invite' + : 'Connect & Sync', onClick: handleSubmit, disabled: !canSubmit || isCreating, }} diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.tsx new file mode 100644 index 00000000000..e8a2fa4cdef --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field.tsx @@ -0,0 +1,129 @@ +'use client' + +import type { ReactNode } from 'react' +import { ButtonGroup, ButtonGroupItem, ChipCombobox, ChipModalField } from '@sim/emcn' +import { + type ConnectorMemberGroupOptions, + decodeConnectorMemberGroupOption, + encodeConnectorMemberGroupOption, +} from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-member-group-options' +import type { ConnectorMeta } from '@/connectors/types' + +/** What the caller chose; `members` may name the option the connector crawls with. */ +export interface ConnectorAccessSelection { + accessMode: 'workspace' | 'members' + credentialGroupId?: string + credentialGroupOptionId?: string +} + +interface ConnectorAccessFieldProps { + connectorConfig: ConnectorMeta + value: ConnectorAccessSelection + onChange: (value: ConnectorAccessSelection) => void + /** From `useConnectorMemberGroupOptions`; shared with the modal so both agree on what is required. */ + groupOptions: ConnectorMemberGroupOptions + /** Only an admin may put a connector into members mode. */ + canAdmin: boolean + disabled?: boolean + /** Whether per-member access may be chosen; false leaves only the way back to workspace access. */ + allowMembers?: boolean + /** + * Whether the connector already syncs per member, so any matching group may + * be chosen, not only when several make the choice necessary. + */ + canRebind?: boolean + /** Rendered under the selection, for a caller that applies the change with its own control. */ + footer?: ReactNode +} + +/** + * The Access section of a connector's settings: sync as the workspace, or + * crawl once per member so each person sees only what the source lets them + * read. Per-member access needs nothing from the admin: a Credential Group is + * found or created for the connector's provider, everyone in the workspace is + * invited, and each person connects their own account. Only a workspace with + * several matching groups is asked which one to use. + */ +export function ConnectorAccessField({ + connectorConfig, + value, + onChange, + groupOptions, + canAdmin, + disabled = false, + allowMembers = true, + canRebind = false, + footer, +}: ConnectorAccessFieldProps) { + if (!groupOptions.supported) return null + + if (!canAdmin) { + if (value.accessMode !== 'members') return null + return ( + + + + Workspace + + + Per member + + + + ) + } + + const selectedValue = + value.accessMode === 'members' && value.credentialGroupId && value.credentialGroupOptionId + ? encodeConnectorMemberGroupOption(value.credentialGroupId, value.credentialGroupOptionId) + : undefined + const { options, needsChoice, isLoading, error } = groupOptions + const showPicker = needsChoice || (canRebind && options.length > 0) + + return ( + +
+ + onChange(mode === 'members' ? { accessMode: 'members' } : { accessMode: 'workspace' }) + } + > + + Workspace + + + Per member + + + + {value.accessMode === 'members' && showPicker && ( + { + const decoded = decodeConnectorMemberGroupOption(next) + if (decoded) onChange({ accessMode: 'members', ...decoded }) + }} + placeholder='Choose which credential group members connect through' + isLoading={isLoading} + disabled={disabled || Boolean(error)} + /> + )} + + {footer} +
+ + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx index 443a7cc4b20..e51d987953a 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connectors-section/connectors-section.tsx @@ -26,11 +26,16 @@ import { Settings, Trash, TriangleAlert, + Users, } from '@sim/emcn/icons' import { createLogger } from '@sim/logger' import { format, formatDistanceToNow, isPast } from 'date-fns' import { consumeOAuthReturnContext, writeOAuthReturnContext } from '@/lib/credentials/client-state' -import { CONNECTOR_SYNC_STALE_LOCK_TTL_MS } from '@/lib/knowledge/connectors/sync-limits' +import { + CONNECTOR_SYNC_STALE_LOCK_TTL_MS, + MEMBER_SYNC_STALE_LOCK_TTL_MS, +} from '@/lib/knowledge/connectors/sync-limits' +import type { MemberSyncStatus } from '@/lib/knowledge/types' import { getCanonicalScopesForProvider, getProviderIdFromServiceId } from '@/lib/oauth' import { getMissingRequiredScopes } from '@/lib/oauth/utils' import { ConnectOAuthModal } from '@/app/workspace/[workspaceId]/components/connect-oauth-modal' @@ -38,7 +43,12 @@ import { EditConnectorModal } from '@/app/workspace/[workspaceId]/knowledge/[id] import { getBlock } from '@/blocks' import { getTileIconColorClass } from '@/blocks/icon-color' import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' -import type { ConnectorData, SyncLogData } from '@/hooks/queries/kb/connectors' +import type { + ConnectorData, + ConnectorMemberSummary, + MemberSyncLogData, + SyncLogData, +} from '@/hooks/queries/kb/connectors' import { isConnectorSyncingOrPending, useConnectorDetail, @@ -77,6 +87,21 @@ const SYNC_IN_FLIGHT_TOOLTIP = { syncing: 'Sync in progress', } as const +/** The member engine's own in-flight states, shown when the connector syncs per member. */ +const MEMBER_SYNC_IN_FLIGHT_TOOLTIP: Partial> = { + pending: 'Member sync queued', + running: 'Syncing members', +} + +/** How each member-engine status reads on the card's badge. */ +const MEMBER_SYNC_STATUS_AS_CONNECTOR_STATUS = { + idle: 'active', + pending: 'pending', + running: 'syncing', + error: 'error', + disabled: 'disabled', +} as const satisfies Record + const CONNECTOR_ACTION_BUTTON_CLASSES = 'size-7 rounded-lg p-0 text-[var(--text-muted)] hover-hover:bg-[var(--surface-active)] hover-hover:text-[var(--text-primary)]' @@ -144,10 +169,18 @@ export function ConnectorsSection({ ) } + const deletingMembersConnector = + connectors.find((connector) => connector.id === deleteTarget)?.accessMode === 'members' + const handleDeleteConnector = () => { if (!deleteTarget) return deleteConnector( - { knowledgeBaseId, connectorId: deleteTarget, deleteDocuments }, + { + knowledgeBaseId, + connectorId: deleteTarget, + /** Documents synced per member have no meaning without their members. */ + deleteDocuments: deleteDocuments || deletingMembersConnector, + }, { onSuccess: () => { setError(null) @@ -216,7 +249,11 @@ export function ConnectorsSection({ }} srTitle='Remove Connector' title='Remove Connector' - text='This will disconnect the source and stop future syncs. Documents already synced will remain in the knowledge base unless you choose to delete them.' + text={ + deletingMembersConnector + ? 'This will disconnect the source, stop future syncs, and delete the documents it synced per member.' + : 'This will disconnect the source and stop future syncs. Documents already synced will remain in the knowledge base unless you choose to delete them.' + } confirm={{ label: 'Remove', onClick: handleDeleteConnector, @@ -224,19 +261,21 @@ export function ConnectorsSection({ pendingLabel: 'Removing...', }} > -
- setDeleteDocuments(checked === true)} - /> - -
+ {!deletingMembersConnector && ( +
+ setDeleteDocuments(checked === true)} + /> + +
+ )} ) @@ -271,8 +310,17 @@ function ConnectorCard({ const connectorDef = CONNECTOR_META_REGISTRY[connector.connectorType] const Icon = connectorDef?.icon const brandBg = getBlock(connector.connectorType)?.bgColor ?? null + /** + * A members-mode connector's content status stays `active` while the member + * engine does the work, so its badge reads the member engine's status. A + * paused or disabled content status still wins: the user set it. + */ + const effectiveStatus = + connector.accessMode === 'members' && connector.status === 'active' + ? MEMBER_SYNC_STATUS_AS_CONNECTOR_STATUS[connector.memberSyncStatus] + : connector.status const statusConfig = - STATUS_CONFIG[connector.status as keyof typeof STATUS_CONFIG] || STATUS_CONFIG.active + STATUS_CONFIG[effectiveStatus as keyof typeof STATUS_CONFIG] || STATUS_CONFIG.active const serviceId = connectorDef?.auth.mode === 'oauth' ? connectorDef.auth.provider : undefined const providerId = serviceId ? getProviderIdFromServiceId(serviceId) : undefined @@ -317,19 +365,39 @@ function ConnectorCard({ expanded ? connector.id : undefined ) const syncLogs = detail?.syncLogs ?? [] + const memberSyncLogs = detail?.memberSyncLogs ?? [] + const members = detail?.members - const canFullResync = Boolean(connectorDef?.rehydrateOnFullSync) + const syncsPerMember = connector.accessMode === 'members' + /** A per-member connector re-hydrates through its members; the content resync has no meaning there. */ + const canFullResync = Boolean(connectorDef?.rehydrateOnFullSync) && !syncsPerMember const syncInFlight = isConnectorSyncingOrPending(connector) const isPaused = connector.status === 'paused' + const memberSyncDisabled = syncsPerMember && connector.memberSyncStatus === 'disabled' /** * A queued sync is what stops a second one being dispatched — the server * rejects it as a conflict anyway, so the button reflects that rather than * running a client-side cooldown timer alongside it. */ - const syncDisabled = syncInFlight || connector.status === 'disabled' || isPaused + const syncDisabled = + syncInFlight || connector.status === 'disabled' || isPaused || memberSyncDisabled const syncTooltip = SYNC_IN_FLIGHT_TOOLTIP[connector.status as keyof typeof SYNC_IN_FLIGHT_TOOLTIP] ?? - (isPaused ? 'Resume to sync' : canFullResync ? 'Sync' : 'Sync now') + (syncsPerMember ? MEMBER_SYNC_IN_FLIGHT_TOOLTIP[connector.memberSyncStatus] : undefined) ?? + (isPaused + ? 'Resume to sync' + : memberSyncDisabled + ? 'Member sync is disabled' + : canFullResync + ? 'Sync' + : syncsPerMember + ? 'Sync members now' + : 'Sync now') + const lastSyncAt = syncsPerMember ? connector.lastMemberSyncAt : connector.lastSyncAt + const nextSyncAt = syncsPerMember ? connector.nextMemberSyncAt : connector.nextSyncAt + const lastSyncError = syncsPerMember + ? (connector.lastMemberSyncError ?? connector.lastSyncError) + : connector.lastSyncError return (
{statusConfig.label} + {syncsPerMember && ( + + Per member + + )}
- {connector.lastSyncAt && ( - Last sync: {format(new Date(connector.lastSyncAt), 'MMM d, h:mm a')} + {lastSyncAt && ( + Last sync: {format(new Date(lastSyncAt), 'MMM d, h:mm a')} )} - {connector.lastSyncDocCount !== null && ( + {!syncsPerMember && connector.lastSyncDocCount !== null && ( <> · {connector.lastSyncDocCount} docs )} - {connector.nextSyncAt && connector.status === 'active' && !syncInFlight && ( + {nextSyncAt && connector.status === 'active' && !syncInFlight && ( <> · Next sync:{' '} - {isPast(new Date(connector.nextSyncAt)) + {isPast(new Date(nextSyncAt)) ? 'pending' - : formatDistanceToNow(new Date(connector.nextSyncAt), { addSuffix: true })} + : formatDistanceToNow(new Date(nextSyncAt), { addSuffix: true })} )} - {connector.lastSyncError && ( + {lastSyncError && ( - {connector.lastSyncError} + {lastSyncError} )} + {connector.accessRewritePending && ( + <> + · + + + Updating access + + + )}
@@ -518,6 +600,22 @@ function ConnectorCard({ + {syncsPerMember && connector.memberSyncStatus === 'disabled' && ( +
+
+
+ + Per-member sync is disabled +
+

+ {connector.lastMemberSyncError ?? 'The connector can no longer sync per member.'}{' '} + Members keep no access until it is fixed; switch the connector's access to re-enable + it. +

+
+
+ )} + {connector.status === 'disabled' && (
@@ -599,7 +697,11 @@ function ConnectorCard({ {expanded && (
- + {syncsPerMember ? ( + + ) : ( + + )}
)} @@ -802,3 +904,130 @@ export function SyncHistory({ logs, isLoading }: SyncHistoryProps) {
) } + +function getMemberSyncLogState(log: MemberSyncLogData, now: number): SyncLogState { + switch (log.status) { + case 'completed': + return 'completed' + case 'failed': + return 'failed' + case 'started': { + const ageMs = now - new Date(log.startedAt).getTime() + return ageMs > MEMBER_SYNC_STALE_LOCK_TTL_MS ? 'interrupted' : 'running' + } + default: { + const exhaustive: never = log.status + return exhaustive + } + } +} + +interface MemberSyncHistoryProps { + logs: MemberSyncLogData[] + members: ConnectorMemberSummary | undefined + isLoading: boolean +} + +/** + * The per-member run history: who was crawled, what changed, and how the + * membership stands. A run that ended with members still due re-dispatches + * itself, so several short rows in a row are one drain, not a fault. + */ +function MemberSyncHistory({ logs, members, isLoading }: MemberSyncHistoryProps) { + if (isLoading) { + return ( +
+ + Loading member sync history… +
+ ) + } + + const now = Date.now() + + return ( +
+ {members && ( +
+ + + {members.active} connected + {members.suspended > 0 && ` · ${members.suspended} need reconnecting`} + {members.stale > 0 && ` · ${members.stale} not synced recently`} + +
+ )} + {logs.length === 0 ? ( +

+ No member sync history yet. +

+ ) : ( +
+ {logs.map((log) => { + const state = getMemberSyncLogState(log, now) + const changes = log.docsAdded + log.docsUpdated + log.docsTombstoned + log.docsPurged + return ( +
+
+ {state === 'running' ? ( + + ) : state === 'interrupted' ? ( + + ) : state === 'failed' ? ( + + ) : ( + + )} +
+
+
+ {format(new Date(log.startedAt), 'MMM d, h:mm a')} + {state === 'completed' && ( + + {log.membersCompleted + log.membersIncomplete + log.membersFailed} member + {log.membersCompleted + log.membersIncomplete + log.membersFailed === 1 + ? '' + : 's'} + {log.membersFailed > 0 && ( + + {' '} + · {log.membersFailed} failed + + )} + {changes > 0 ? ( + <> + {log.docsAdded > 0 && ( + +{log.docsAdded} + )} + {log.docsUpdated > 0 && ( + ~{log.docsUpdated} + )} + {log.docsTombstoned + log.docsPurged > 0 && ( + + {' '} + -{log.docsTombstoned + log.docsPurged} + + )} + + ) : ( + ' · no changes' + )} + + )} + {state === 'running' && In progress…} + {state === 'interrupted' && ( + Interrupted + )} +
+ {state === 'failed' && log.errorMessage && ( + {log.errorMessage} + )} +
+
+ ) + })} +
+ )} +
+ ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/consts.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/consts.ts index 4ab8aff2a96..6551b3be525 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/consts.ts +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/consts.ts @@ -1,3 +1,7 @@ +/** Under the account picker of a per-member connector, whose account only browses. */ +export const BROWSE_WITH_HINT = + 'Only used to pick folders and spaces below. The connector syncs as each member, not as this account.' + export const SYNC_INTERVALS = [ { label: 'Live', value: 5, requiresMax: true }, { label: 'Every hour', value: 60, requiresMax: false }, diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx index c2e13149394..1caa439d744 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/edit-connector-modal.tsx @@ -5,6 +5,7 @@ import { Button, ButtonGroup, ButtonGroupItem, + ChipCombobox, ChipModal, ChipModalBody, ChipModalError, @@ -12,21 +13,36 @@ import { ChipModalFooter, ChipModalHeader, ChipModalTabs, + type ComboboxOption, Skeleton, Tooltip, } from '@sim/emcn' import { RefreshCw, SquareArrowUpRight } from '@sim/emcn/icons' import { createLogger } from '@sim/logger' +import { useParams } from 'next/navigation' +import { getProviderIdFromServiceId, type OAuthProvider } from '@/lib/oauth' +import { + ConnectorAccessField, + type ConnectorAccessSelection, +} from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connector-access-field/connector-access-field' import { ConnectorConfigFields } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connector-config-fields' import { hasWorkspaceMaxConnectorAccess } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/connector-entitlements' -import { SYNC_INTERVALS } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/consts' +import { + BROWSE_WITH_HINT, + SYNC_INTERVALS, +} from '@/app/workspace/[workspaceId]/knowledge/[id]/components/consts' import { MaxBadge } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/max-badge' import type { ConfigFieldMap, ConfigFieldValue, } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields' import { useConnectorConfigFields } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields' +import { + memberCapFieldIds, + useConnectorMemberGroupOptions, +} from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-member-group-options' import { useWorkspaceHostContext } from '@/app/workspace/[workspaceId]/providers/workspace-host-provider' +import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' import { withBrandIcon } from '@/blocks/brand-icon' import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' import type { ConnectorConfigField, ConnectorMeta } from '@/connectors/types' @@ -36,7 +52,9 @@ import { useExcludeConnectorDocument, useRestoreConnectorDocument, useUpdateConnector, + useUpdateConnectorAccess, } from '@/hooks/queries/kb/connectors' +import { useOAuthCredentials } from '@/hooks/queries/oauth/oauth-credentials' const logger = createLogger('EditConnectorModal') @@ -45,6 +63,27 @@ const INTERNAL_CONFIG_KEYS = new Set(['tagSlotMapping', 'disabledTagIds', '_cano const CANONICAL_MODES_KEY = '_canonicalModes' +/** The access a connector row currently has, as the Access field edits it. */ +function currentAccess(connector: ConnectorData): ConnectorAccessSelection { + if (connector.accessMode === 'members') { + return { + accessMode: 'members', + credentialGroupId: connector.credentialGroupId ?? undefined, + credentialGroupOptionId: connector.credentialGroupOptionId ?? undefined, + } + } + return { accessMode: 'workspace' } +} + +function accessChanged(current: ConnectorAccessSelection, next: ConnectorAccessSelection): boolean { + if (current.accessMode !== next.accessMode) return true + if (next.accessMode === 'workspace') return false + return ( + current.credentialGroupId !== next.credentialGroupId || + current.credentialGroupOptionId !== next.credentialGroupOptionId + ) +} + function readPersistedCanonicalModes( sourceConfig: Record ): Record { @@ -130,6 +169,8 @@ export function EditConnectorModal({ const [activeTab, setActiveTab] = useState('settings') const [syncInterval, setSyncInterval] = useState(connector.syncIntervalMinutes) + const [access, setAccess] = useState(() => currentAccess(connector)) + const [workspaceCredentialId, setWorkspaceCredentialId] = useState(null) const [error, setError] = useState(null) /** @@ -191,11 +232,41 @@ export function EditConnectorModal({ initialCanonicalModes, }) - const { ownerBilling } = useWorkspaceHostContext() - const { mutate: updateConnector, isPending: isSaving } = useUpdateConnector() + const { ownerBilling, features } = useWorkspaceHostContext() + const { canAdmin } = useUserPermissionsContext() + const { workspaceId } = useParams<{ workspaceId: string }>() + const { mutate: updateConnector, isPending: isSavingSettings } = useUpdateConnector() + const { mutate: updateAccess, isPending: isSwitchingAccess } = useUpdateConnectorAccess() + const isSaving = isSavingSettings || isSwitchingAccess + /** + * The field shows where the flag is on. A connector already syncing per + * member keeps it where the flag has since been turned off, so an admin can + * still bring it back to workspace mode; per-member cannot be re-chosen. + */ + const memberAccessAvailable = features?.knowledgeMemberAccess === true + const showAccessField = memberAccessAvailable || connector.accessMode === 'members' const hasMaxAccess = hasWorkspaceMaxConnectorAccess(ownerBilling) + const accessDirty = accessChanged(currentAccess(connector), access) + const groupOptions = useConnectorMemberGroupOptions({ + workspaceId, + connectorConfig, + enabled: canAdmin && memberAccessAvailable, + }) + /** Leaving members mode needs the credential the connector syncs as from then on. */ + const needsWorkspaceCredential = + accessDirty && access.accessMode === 'workspace' && connector.accessMode === 'members' + const accessComplete = + !accessDirty || + (access.accessMode === 'members' + ? !groupOptions.needsChoice || Boolean(access.credentialGroupOptionId) + : !needsWorkspaceCredential || Boolean(workspaceCredentialId)) + /** A disabled member sync is re-enabled by applying the current binding again. */ + const canReenableMemberSync = + !accessDirty && connector.accessMode === 'members' && connector.memberSyncStatus === 'disabled' + const hiddenCapFieldIds = memberCapFieldIds(connectorConfig, access.accessMode) + const persistedCanonicalModes = useMemo( () => readPersistedCanonicalModes(connector.sourceConfig), [connector.sourceConfig] @@ -253,9 +324,7 @@ export function EditConnectorModal({ updateConnector( { knowledgeBaseId, connectorId: connector.id, updates }, { - onSuccess: () => { - onOpenChange(false) - }, + onSuccess: () => onOpenChange(false), onError: (err) => { logger.error('Failed to update connector', { error: err.message }) setError(err.message) @@ -264,6 +333,40 @@ export function EditConnectorModal({ ) } + /** + * The mode switch is its own admin operation: it rewrites document access + * and queues a run of the other engine, so it is applied on its own rather + * than folded into a settings save that would race the run it starts. + */ + const handleApplyAccess = () => { + setError(null) + updateAccess( + { + knowledgeBaseId, + connectorId: connector.id, + access: + access.accessMode === 'members' + ? { + accessMode: 'members', + credentialGroupId: access.credentialGroupId, + credentialGroupOptionId: access.credentialGroupOptionId, + } + : { + accessMode: 'workspace', + credentialId: workspaceCredentialId ?? undefined, + }, + }, + { + /** The connector prop is a snapshot; closing hands the refreshed row to the next open. */ + onSuccess: () => onOpenChange(false), + onError: (err) => { + logger.error('Failed to switch connector access', { error: err.message }) + setError(err.message) + }, + } + ) + } + const displayName = connectorConfig?.name ?? connector.connectorType const Icon = connectorConfig?.icon @@ -293,18 +396,35 @@ export function EditConnectorModal({ {activeTab === 'settings' ? ( isFieldVisible(field) && !hiddenCapFieldIds.has(field.id)} syncInterval={syncInterval} setSyncInterval={setSyncInterval} hasMaxAccess={hasMaxAccess} isSaving={isSaving} error={error} + access={access} + onAccessChange={setAccess} + canAdmin={canAdmin} + showAccessField={showAccessField} + allowMembers={memberAccessAvailable} + groupOptions={groupOptions} + canReenableMemberSync={canReenableMemberSync} + accessDirty={accessDirty} + accessComplete={accessComplete} + isSwitchingAccess={isSwitchingAccess} + onApplyAccess={handleApplyAccess} + onResetAccess={() => setAccess(currentAccess(connector))} + workspaceId={workspaceId} + needsWorkspaceCredential={needsWorkspaceCredential} + workspaceCredentialId={workspaceCredentialId} + onWorkspaceCredentialChange={setWorkspaceCredentialId} /> ) : ( @@ -317,7 +437,8 @@ export function EditConnectorModal({ primaryAction={{ label: isSaving ? 'Saving…' : 'Save', onClick: handleSave, - disabled: !hasChanges || isSaving, + /** An open access change is applied by its own control, never folded into Save. */ + disabled: !hasChanges || accessDirty || isSaving, }} /> )} @@ -327,6 +448,8 @@ export function EditConnectorModal({ interface SettingsTabProps { connectorConfig: ConnectorMeta | null + /** The mode the connector is saved in, which the draft `access` may differ from. */ + persistedAccessMode: 'workspace' | 'members' sourceConfig: ConfigFieldMap credentialId: string | null canonicalGroups: Map @@ -339,10 +462,27 @@ interface SettingsTabProps { hasMaxAccess: boolean isSaving: boolean error: string | null + access: ConnectorAccessSelection + onAccessChange: (access: ConnectorAccessSelection) => void + canAdmin: boolean + showAccessField: boolean + allowMembers: boolean + groupOptions: ReturnType + canReenableMemberSync: boolean + accessDirty: boolean + accessComplete: boolean + isSwitchingAccess: boolean + onApplyAccess: () => void + onResetAccess: () => void + workspaceId: string + needsWorkspaceCredential: boolean + workspaceCredentialId: string | null + onWorkspaceCredentialChange: (credentialId: string) => void } function SettingsTab({ connectorConfig, + persistedAccessMode, sourceConfig, credentialId, canonicalGroups, @@ -355,14 +495,141 @@ function SettingsTab({ hasMaxAccess, isSaving, error, + access, + onAccessChange, + canAdmin, + showAccessField, + allowMembers, + groupOptions, + canReenableMemberSync, + accessDirty, + accessComplete, + isSwitchingAccess, + onApplyAccess, + onResetAccess, + workspaceId, + needsWorkspaceCredential, + workspaceCredentialId, + onWorkspaceCredentialChange, }: SettingsTabProps) { + const providerId = + connectorConfig?.auth.mode === 'oauth' + ? (getProviderIdFromServiceId(connectorConfig.auth.provider) as OAuthProvider) + : null + const syncsPerMember = access.accessMode === 'members' + /** Staying per member but through a different group. */ + const isRebind = accessDirty && persistedAccessMode === 'members' && syncsPerMember + const { data: rawCredentials = [], isLoading: credentialsLoading } = useOAuthCredentials( + providerId ?? undefined, + { enabled: (needsWorkspaceCredential || syncsPerMember) && Boolean(providerId), workspaceId } + ) + const [browseCredentialId, setBrowseCredentialId] = useState(null) + /** A per-member connector has no credential of its own; the admin's account browses the source. */ + const selectorCredentialId = syncsPerMember ? browseCredentialId : credentialId + const credentialOptions = useMemo( + () => + rawCredentials + .filter((credential) => credential.type !== 'service_account') + .map((credential) => ({ + label: credential.name || credential.provider, + value: credential.id, + })), + [rawCredentials] + ) + return ( <> + {connectorConfig && connectorConfig.auth.mode === 'oauth' && showAccessField && ( + +
+ +
+

+ Members and their documents are kept; the next sync restores their access. +

+
+ ) : accessDirty ? ( +
+ {needsWorkspaceCredential && ( + <> + + {!credentialsLoading && credentialOptions.length === 0 && ( +

+ Connect a {connectorConfig.name} account in Integrations first. +

+ )} + + )} +
+ + +
+

+ {isRebind + ? 'Members of the previous group lose access; members of the new group are invited to connect.' + : access.accessMode === 'members' + ? 'Everyone in the workspace is invited to connect their account. Documents stay hidden until members connect and sync; listing caps are cleared.' + : 'Every workspace member can read every synced document once the next sync completes.'} +

+
+ ) : undefined + } + /> + )} + + {connectorConfig && syncsPerMember && ( + + + + )} + {connectorConfig && ( { + return new Set( + accessMode === 'members' ? (connectorConfig?.permissionScopedListing?.capFieldIds ?? []) : [] + ) +} + +interface UseConnectorMemberGroupOptionsInput { + workspaceId: string + connectorConfig: ConnectorMeta | null + /** False leaves the query off and reports no options, for a viewer who cannot choose anyway. */ + enabled: boolean +} + +export interface ConnectorMemberGroupOptions { + /** Every active option in the workspace collecting the connector's accounts, as combobox entries. */ + options: ComboboxOption[] + /** Whether the connector's provider can be collected through a Credential Group at all. */ + supported: boolean + /** More than one candidate: the admin has to say which, or the server refuses the ambiguity. */ + needsChoice: boolean + isLoading: boolean + error: Error | null +} + +/** + * The Credential Group options a per-member connector could sync through. + * One source for the Access field, which renders them, and the modals, which + * must not submit while a choice between several is still open. + */ +export function useConnectorMemberGroupOptions({ + workspaceId, + connectorConfig, + enabled, +}: UseConnectorMemberGroupOptionsInput): ConnectorMemberGroupOptions { + const provider = connectorConfig ? connectorMemberGroupProvider(connectorConfig) : null + const providerId = provider ? getCredentialGroupProviderId(provider) : null + const { + data: settings, + isLoading, + error, + } = useCredentialGroups(enabled && provider ? workspaceId : undefined) + + const options = useMemo(() => { + if (!settings || !providerId) return [] + const entries: ComboboxOption[] = [] + for (const group of settings.credentialGroups) { + if (group.status !== 'active') continue + for (const option of group.options) { + if (option.status !== 'active') continue + if (!isCredentialGroupProvider(option.provider)) continue + if (getCredentialGroupProviderId(option.provider) !== providerId) continue + entries.push({ + label: `${group.name} · ${option.label}`, + value: encodeConnectorMemberGroupOption(group.id, option.id), + }) + } + } + return entries + }, [settings, providerId]) + + return { + options, + supported: provider !== null, + needsChoice: options.length > 1, + isLoading, + error: error ?? null, + } +} diff --git a/apps/sim/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section.tsx b/apps/sim/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section.tsx new file mode 100644 index 00000000000..43ee669f90c --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section.tsx @@ -0,0 +1,98 @@ +'use client' + +import { useMemo } from 'react' +import { Button } from '@sim/emcn' +import { connectorDisplayName } from '@/lib/sim-search/connectors' +import { IntegrationTile } from '@/app/workspace/[workspaceId]/integrations/components/integrations-showcase' +import { + RESOURCE_LIST_STACK, + SettingsResourceRow, +} from '@/app/workspace/[workspaceId]/settings/components/settings-resource-row' +import { SettingsSection } from '@/app/workspace/[workspaceId]/settings/components/settings-section/settings-section' +import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' +import { memberConnectorKeys, type WorkspaceMemberConnector } from '@/hooks/queries/kb/connectors' +import { + CONNECTABLE_MEMBERSHIPS, + describeMembership, + enrollmentActionLabel, + useMemberEnrollment, +} from '@/hooks/use-member-enrollment' + +const SHARED_WITH_YOU_LABEL = 'Shared with you' + +interface MemberConnectorsSectionProps { + workspaceId: string + /** The per-member connectors to show, already narrowed by the page's search. */ + connectors: WorkspaceMemberConnector[] +} + +/** + * The knowledge bases whose connectors sync per member, and where the viewer + * stands with each. Connecting here is the same enrollment the knowledge base + * page offers, so a person can do it from whichever surface they are on. + */ +export function MemberConnectorsSection({ workspaceId, connectors }: MemberConnectorsSectionProps) { + const connectedConnectorIds = useMemo( + () => + new Set( + connectors + .filter((connector) => connector.viewerMembership === 'connected') + .map((connector) => connector.connectorId) + ), + [connectors] + ) + const membershipQueryKeys = useMemo(() => [memberConnectorKeys.list(workspaceId)], [workspaceId]) + const { connect, isAwaiting, isPending, error } = useMemberEnrollment({ + membershipQueryKeys, + connectedConnectorIds, + }) + + if (connectors.length === 0) return null + + return ( + <> + +
+ {connectors.map((connector) => { + const meta = CONNECTOR_META_REGISTRY[connector.connectorType] + const name = connectorDisplayName(connector.connectorType) + const waiting = isAwaiting(connector.connectorId) + const state = + describeMembership({ + membership: connector.viewerMembership, + memberSyncStatus: connector.memberSyncStatus, + waiting, + name, + }) ?? 'Connected.' + return ( + + ) : undefined + } + title={name} + description={`${connector.knowledgeBaseName} · ${state}`} + trailing={ + CONNECTABLE_MEMBERSHIPS.has(connector.viewerMembership) ? ( + + ) : undefined + } + /> + ) + })} +
+ + {error &&

{error}

} + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/search/error.tsx b/apps/sim/app/workspace/[workspaceId]/search/error.tsx new file mode 100644 index 00000000000..d4520d2d64f --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/search/error.tsx @@ -0,0 +1,15 @@ +'use client' + +import { type ErrorBoundaryProps, ErrorState } from '@/app/workspace/[workspaceId]/components' + +export default function SearchError({ error, reset }: ErrorBoundaryProps) { + return ( + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/search/page.tsx b/apps/sim/app/workspace/[workspaceId]/search/page.tsx new file mode 100644 index 00000000000..1623fd074e2 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/search/page.tsx @@ -0,0 +1,30 @@ +import { Suspense } from 'react' +import type { Metadata } from 'next' +import { IntegrationTabsHeader } from '@/app/workspace/[workspaceId]/components' +import { Search } from '@/app/workspace/[workspaceId]/search/search' + +export const metadata: Metadata = { + title: 'Search', +} + +/** + * Sim Search page entry. `Search` reads URL query params via nuqs (which uses + * `useSearchParams` internally), so it must sit under a Suspense boundary. The + * fallback renders the real page chrome (background + tab header) so a suspend + * never shows a blank frame. + */ +export default async function SearchPage({ params }: { params: Promise<{ workspaceId: string }> }) { + const { workspaceId } = await params + + return ( + + + + } + > + + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/search/search-params.ts b/apps/sim/app/workspace/[workspaceId]/search/search-params.ts new file mode 100644 index 00000000000..c55f7709913 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/search/search-params.ts @@ -0,0 +1,17 @@ +import { parseAsString } from 'nuqs/server' + +/** + * `search` filters the Sim Search connector list by name and description. The + * input is controlled directly by the instant nuqs value; only its URL write is + * debounced via `useDebouncedSearchSetter` — never written on every keystroke. + */ +export const connectorSearchParam = { + key: 'search', + parser: parseAsString.withDefault(''), +} as const + +/** Search is filter view-state: clean URLs, no back-stack churn. */ +export const connectorSearchUrlKeys = { + history: 'replace', + clearOnDefault: true, +} as const diff --git a/apps/sim/app/workspace/[workspaceId]/search/search.test.tsx b/apps/sim/app/workspace/[workspaceId]/search/search.test.tsx new file mode 100644 index 00000000000..b76e3ea6395 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/search/search.test.tsx @@ -0,0 +1,225 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, describe, expect, it, vi } from 'vitest' + +const { mockConnect, mockConnectSource, mockFeatures } = vi.hoisted(() => ({ + mockConnect: vi.fn(), + mockConnectSource: vi.fn(), + mockFeatures: vi.fn(), +})) + +vi.mock('next/navigation', () => ({ + useParams: () => ({ workspaceId: 'workspace-1' }), +})) +vi.mock('@/app/workspace/[workspaceId]/providers/workspace-host-provider', () => ({ + useWorkspaceHostContext: () => ({ features: mockFeatures() }), +})) +vi.mock('nuqs', () => ({ + useQueryState: () => ['', vi.fn()], +})) +vi.mock('@/hooks/use-debounced-search-setter', () => ({ + useDebouncedSearchSetter: (write: (value: string) => void) => write, +})) +vi.mock('@/hooks/queries/workspace', () => ({ + useWorkspacePermissionsQuery: () => ({ data: { viewer: { isAdmin: true } } }), +})) +vi.mock('@/hooks/use-permission-config', () => ({ + usePermissionConfig: () => ({ + integrationAvailability: new Map([ + ['slack', { state: 'limited', oauthAvailable: false }], + ['jira', { state: 'available', oauthAvailable: true }], + ]), + }), +})) +vi.mock('@/app/workspace/[workspaceId]/integrations/hooks/use-scroll-restoration', () => ({ + useScrollRestoration: () => undefined, +})) +vi.mock('@/app/workspace/[workspaceId]/components', () => ({ + IntegrationTabsHeader: () => null, +})) +vi.mock('@/blocks', () => ({ getBlock: () => undefined })) +vi.mock('@/lib/integrations', () => ({ + blockTypeToIconMap: {}, + resolveCredentialDisplay: () => ({ icon: () => null, blockType: 'confluence', subtitle: 'Sub' }), +})) + +vi.mock('@/lib/sim-search/connectors', () => { + const icon = () => null + const connector = (type: string, name: string, description: string, personal: boolean) => ({ + type, + meta: { + id: type, + name, + description, + icon, + auth: { mode: 'oauth', provider: type }, + permissionScopedListing: personal ? { capFieldIds: [] } : undefined, + configFields: personal ? [] : [{ id: 'domain', required: true }], + }, + providerId: type, + providerIds: [type], + requiredScopes: [], + serviceName: name, + serviceIcon: icon, + blockType: type, + setupFields: [], + }) + const isSearchConnectorAvailable = ( + candidate: { blockType: string }, + availability: ReadonlyMap + ) => availability.get(candidate.blockType)?.oauthAvailable ?? true + return { + SIM_SEARCH_KNOWLEDGE_BASE_NAME: 'Sim Search', + canConnectPersonally: (meta: { permissionScopedListing?: unknown }) => + Boolean(meta.permissionScopedListing), + connectorDisplayName: (connectorType: string) => connectorType, + isSearchConnectorAvailable, + searchConnectorUnavailableReason: ( + candidate: { blockType: string; meta: { name: string } }, + availability: ReadonlyMap, + context: { memberAccessAvailable: boolean; hasConnection: boolean; canCreate: boolean } + ) => + !isSearchConnectorAvailable(candidate, availability) + ? `${candidate.meta.name} is unavailable in this deployment` + : !context.memberAccessAvailable + ? 'Per-member access is not available in this workspace' + : !context.hasConnection && !context.canCreate + ? `Ask a workspace admin to connect ${candidate.meta.name} first` + : null, + SEARCH_CONNECTORS: [ + connector('google_drive', 'Google Drive', 'Sync Drive files', true), + connector('confluence', 'Confluence', 'Sync Confluence pages', false), + connector('slack', 'Slack', 'Sync Slack messages', true), + ], + } +}) + +vi.mock('@/hooks/queries/kb/connectors', () => ({ + memberConnectorKeys: { list: (workspaceId?: string) => ['member-connectors', workspaceId] }, + useWorkspaceMemberConnectors: () => ({ + isPending: false, + data: [ + { + knowledgeBaseId: 'kb-search', + knowledgeBaseName: 'Sim Search', + connectorId: 'conn-drive', + connectorType: 'google_drive', + memberSyncStatus: 'idle', + viewerMembership: 'connected', + viewerDocumentCount: 12, + }, + { + knowledgeBaseId: 'kb-sales', + knowledgeBaseName: 'Sales', + connectorId: 'conn-sales-drive', + connectorType: 'google_drive', + memberSyncStatus: 'idle', + viewerMembership: 'invited', + viewerDocumentCount: 0, + }, + ], + }), +})) +vi.mock('@/hooks/use-member-enrollment', async () => { + const actual = await vi.importActual( + '@/hooks/use-member-enrollment' + ) + return { + CONNECTABLE_MEMBERSHIPS: actual.CONNECTABLE_MEMBERSHIPS, + describeMembership: actual.describeMembership, + enrollmentActionLabel: actual.enrollmentActionLabel, + useMemberEnrollment: () => ({ + connect: mockConnect, + connectSource: mockConnectSource, + connectSearchSource: ( + workspaceId: string, + connector: { type: string }, + connection: { knowledgeBaseId: string; connectorId: string } | undefined + ) => + connection + ? mockConnect(connection.knowledgeBaseId, connection.connectorId) + : mockConnectSource(workspaceId, connector.type), + setupConnector: null, + closeSetup: () => {}, + isAwaiting: () => false, + isAwaitingSource: () => false, + isPending: false, + error: null, + }), + } +}) +vi.mock('@/connectors/registry', () => ({ + CONNECTOR_META_REGISTRY: { google_drive: { name: 'Google Drive', icon: () => null } }, +})) + +import { Search } from '@/app/workspace/[workspaceId]/search/search' + +let root: Root | null = null +let container: HTMLDivElement | null = null + +function mount(features: { knowledgeMemberAccess?: boolean } = { knowledgeMemberAccess: true }) { + mockFeatures.mockReturnValue({ credentialGroups: true, ...features }) + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) + act(() => root?.render()) +} + +function sectionLabels(): string[] { + return Array.from(container?.querySelectorAll('section > div > span') ?? []).map( + (node) => node.textContent ?? '' + ) +} + +function buttons(): HTMLButtonElement[] { + return Array.from(container?.querySelectorAll('button') ?? []) +} + +afterEach(() => { + if (root) act(() => root?.unmount()) + container?.remove() + root = null + container = null + mockConnect.mockReset() + mockConnectSource.mockReset() +}) + +describe('Search', () => { + it('shows each source with the viewer’s own connection state', () => { + mount() + + expect(sectionLabels()).toEqual(['Sim Search Connectors', 'Shared with you']) + const text = container?.textContent ?? '' + expect(text).toContain('Connected · 12 documents') + expect(text).toContain('Set up by a workspace admin from a knowledge base.') + expect(text).toContain('Slack is unavailable in this deployment') + expect(text).toContain('Sales') + }) + + it('connects a source nobody has connected yet through its per-member connector', () => { + mount() + + const connect = buttons().find((button) => button.textContent === 'Connect') + expect(connect).toBeDefined() + act(() => { + connect?.dispatchEvent(new MouseEvent('click', { bubbles: true, button: 0 })) + }) + + expect(mockConnect).toHaveBeenCalledWith('kb-sales', 'conn-sales-drive') + expect(mockConnectSource).not.toHaveBeenCalled() + }) + + it('offers no connection while per-member access is unavailable in the workspace', () => { + mount({ knowledgeMemberAccess: false }) + + expect(sectionLabels()).toEqual(['Sim Search Connectors']) + const text = container?.textContent ?? '' + expect(text).toContain('Per-member access is not available in this workspace') + expect(text).not.toContain('Connected · 12 documents') + expect(buttons().find((button) => button.textContent === 'Connect')).toBeUndefined() + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/search/search.tsx b/apps/sim/app/workspace/[workspaceId]/search/search.tsx new file mode 100644 index 00000000000..2d99116f189 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/search/search.tsx @@ -0,0 +1,296 @@ +'use client' + +import { useMemo, useRef } from 'react' +import { Button, ChipInput } from '@sim/emcn' +import { Search as SearchIcon } from '@sim/emcn/icons' +import { useParams } from 'next/navigation' +import { useQueryState } from 'nuqs' +import { + canConnectPersonally, + connectorDisplayName, + SEARCH_CONNECTORS, + type SearchConnector, + SIM_SEARCH_KNOWLEDGE_BASE_NAME, + searchConnectorUnavailableReason, +} from '@/lib/sim-search/connectors' +import { IntegrationTabsHeader } from '@/app/workspace/[workspaceId]/components' +import { SourceSetupModal } from '@/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal' +import { IntegrationSection } from '@/app/workspace/[workspaceId]/integrations/components/integration-section' +import { IntegrationTile } from '@/app/workspace/[workspaceId]/integrations/components/integrations-showcase' +import { useScrollRestoration } from '@/app/workspace/[workspaceId]/integrations/hooks/use-scroll-restoration' +import { useWorkspaceHostContext } from '@/app/workspace/[workspaceId]/providers/workspace-host-provider' +import { MemberConnectorsSection } from '@/app/workspace/[workspaceId]/search/components/member-connectors-section/member-connectors-section' +import { + connectorSearchParam, + connectorSearchUrlKeys, +} from '@/app/workspace/[workspaceId]/search/search-params' +import { SettingsEmptyState } from '@/app/workspace/[workspaceId]/settings/components/settings-empty-state' +import { SettingsResourceRow } from '@/app/workspace/[workspaceId]/settings/components/settings-resource-row' +import { + memberConnectorKeys, + useWorkspaceMemberConnectors, + type WorkspaceMemberConnector, +} from '@/hooks/queries/kb/connectors' +import { useWorkspacePermissionsQuery } from '@/hooks/queries/workspace' +import { useDebouncedSearchSetter } from '@/hooks/use-debounced-search-setter' +import { + CONNECTABLE_MEMBERSHIPS, + describeMembership, + enrollmentActionLabel, + useMemberEnrollment, +} from '@/hooks/use-member-enrollment' +import { usePermissionConfig } from '@/hooks/use-permission-config' + +const EMPTY_MEMBER_CONNECTORS: WorkspaceMemberConnector[] = [] +const CONNECTORS_LABEL = 'Sim Search Connectors' +const NEEDS_KNOWLEDGE_BASE_SETUP = 'Set up by a workspace admin from a knowledge base.' + +/** What a source row says once the viewer's own indexing has settled. */ +function connectedDescription(connector: WorkspaceMemberConnector): string { + const count = connector.viewerDocumentCount + return count === 1 ? 'Connected · 1 document' : `Connected · ${count} documents` +} + +interface SourceRowProps { + connector: SearchConnector + /** The Sim Search per-member connector for this source, once anyone has connected it. */ + connection: WorkspaceMemberConnector | undefined + /** Why the source cannot be connected here, shown in place of its state; null when it can. */ + unavailableReason: string | null + waiting: boolean + isPending: boolean + onConnect: () => void +} + +/** + * One Sim Search source: what the viewer's connection is doing (indexing, + * how many documents they can read, what to do next) and the one action open + * to them. A source nobody has connected yet offers Connect, which creates its + * connector and enrolls the viewer in one step. + */ +function SourceRow({ + connector, + connection, + unavailableReason, + waiting, + isPending, + onConnect, +}: SourceRowProps) { + const unavailable = unavailableReason !== null + const personal = canConnectPersonally(connector.meta) + const membership = connection?.viewerMembership + const state = connection + ? (describeMembership({ + membership: connection.viewerMembership, + memberSyncStatus: connection.memberSyncStatus, + waiting, + name: connector.meta.name, + }) ?? connectedDescription(connection)) + : waiting + ? `Finish connecting your ${connector.meta.name} account in the other tab.` + : connector.meta.description + const description = unavailableReason ?? (personal ? state : NEEDS_KNOWLEDGE_BASE_SETUP) + const connectable = + !unavailable && !waiting && personal && (!membership || CONNECTABLE_MEMBERSHIPS.has(membership)) + return ( + } + title={connector.meta.name} + description={description} + disabled={unavailable || !personal} + trailing={ + connectable ? ( + + ) : undefined + } + /> + ) +} + +/** + * The Sim Search catalog: every source a person can connect with one click, + * each row showing where the viewer's own connection stands. Connecting opens + * the enrollment for the workspace's Sim Search knowledge base, and indexing + * starts on its own once the account is linked; documents count up here as + * they land. Per-member connectors in other knowledge bases are listed below + * under Shared with you, with the same actions. + */ +export function Search() { + const scrollContainerRef = useRef(null) + const params = useParams() + const workspaceId = (params?.workspaceId as string) || '' + const { integrationAvailability } = usePermissionConfig() + const { features } = useWorkspaceHostContext() + /** + * Judged by the workspace, as the server judges it: with per-member access + * off, every connect is refused, so the rows say so instead of offering + * one and the memberships are not fetched. + */ + const memberAccessAvailable = features?.knowledgeMemberAccess === true + const { data: workspacePermissions } = useWorkspacePermissionsQuery(workspaceId) + /** The first connect of a source turns it on for the workspace, which takes an admin. */ + const canCreate = workspacePermissions?.viewer?.isAdmin ?? false + + const [searchTerm, setSearchTermParam] = useQueryState(connectorSearchParam.key, { + ...connectorSearchParam.parser, + ...connectorSearchUrlKeys, + }) + /** + * The input binds to the instant nuqs value; only the URL write is debounced. + * Filtering reads the same instant value: it is a cheap in-memory pass over a + * small static list, which is exactly the case the url-state rule permits. + */ + const setSearchTerm = useDebouncedSearchSetter(setSearchTermParam) + + const { data: memberConnectorRows, isPending: connectionsPending } = useWorkspaceMemberConnectors( + workspaceId, + { enabled: memberAccessAvailable } + ) + /** Rows cached before the feature went off are not this surface's to show. */ + const memberConnectors = memberAccessAvailable + ? (memberConnectorRows ?? EMPTY_MEMBER_CONNECTORS) + : EMPTY_MEMBER_CONNECTORS + useScrollRestoration(scrollContainerRef, { + ready: !memberAccessAvailable || !connectionsPending, + }) + + /** The Sim Search connection per source; other knowledge bases' connectors keep their own section. */ + const { connectionByType, sharedConnectors } = useMemo(() => { + const connectionByType = new Map() + const sharedConnectors: WorkspaceMemberConnector[] = [] + for (const connector of memberConnectors) { + if ( + connector.knowledgeBaseName === SIM_SEARCH_KNOWLEDGE_BASE_NAME && + !connectionByType.has(connector.connectorType) + ) { + connectionByType.set(connector.connectorType, connector) + } else { + sharedConnectors.push(connector) + } + } + return { connectionByType, sharedConnectors } + }, [memberConnectors]) + const connectedConnectorIds = useMemo( + () => + new Set( + memberConnectors + .filter((connector) => connector.viewerMembership === 'connected') + .map((connector) => connector.connectorId) + ), + [memberConnectors] + ) + const membershipQueryKeys = useMemo(() => [memberConnectorKeys.list(workspaceId)], [workspaceId]) + const { + connectSource, + connectSearchSource, + setupConnector, + closeSetup, + isAwaiting, + isAwaitingSource, + isPending, + error, + } = useMemberEnrollment({ + membershipQueryKeys, + connectedConnectorIds, + }) + + const normalizedSearch = searchTerm.trim().toLowerCase() + const visibleConnectors = normalizedSearch + ? SEARCH_CONNECTORS.filter( + (connector) => + connector.meta.name.toLowerCase().includes(normalizedSearch) || + connector.meta.description.toLowerCase().includes(normalizedSearch) + ) + : SEARCH_CONNECTORS + const visibleSharedConnectors = normalizedSearch + ? sharedConnectors.filter((connector) => + [connectorDisplayName(connector.connectorType), connector.knowledgeBaseName].some((text) => + text.toLowerCase().includes(normalizedSearch) + ) + ) + : sharedConnectors + + const showNoResults = + Boolean(normalizedSearch) && + visibleConnectors.length === 0 && + visibleSharedConnectors.length === 0 + + return ( +
+ +
+
+ setSearchTerm(e.target.value)} + /> + +
+ {visibleConnectors.length > 0 && ( + + {visibleConnectors.map((connector) => { + const connection = connectionByType.get(connector.type) + return ( + connectSearchSource(workspaceId, connector, connection)} + /> + ) + })} + + )} + + {memberAccessAvailable && ( + + )} + + {error &&

{error}

} + {setupConnector && ( + + connectSource(workspaceId, setupConnector.type, sourceConfig) + } + /> + )} + + {showNoResults && ( + + No connectors found matching “{searchTerm}” + + )} +
+
+
+
+ ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/knowledge-base-selector/knowledge-base-selector.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/knowledge-base-selector/knowledge-base-selector.tsx index 521e70eb7ae..deb7b7494a8 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/knowledge-base-selector/knowledge-base-selector.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/editor/components/sub-block/components/knowledge-base-selector/knowledge-base-selector.tsx @@ -184,6 +184,8 @@ export function KnowledgeBaseSelector({ const label = subBlock.placeholder || (isMultiSelect ? 'Select knowledge bases' : 'Select knowledge base') + const hasMemberScopedSelection = selectedKnowledgeBases.some((kb) => kb.hasMemberScopedConnector) + return (
{/* Selected knowledge bases display (for multi-select) */} @@ -256,6 +258,13 @@ export function KnowledgeBaseSelector({ : undefined } /> + {hasMemberScopedSelection && ( +

+ Documents synced per member are returned only when the person who triggers the run has + connected their account. Scheduled, API, webhook, and chat runs see workspace-visible + documents only. +

+ )}
) } diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/panel.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/panel.tsx index 2876a06a44a..c1522b801cf 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/panel.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/panel.tsx @@ -512,6 +512,8 @@ export const Panel = memo(function Panel() { const handler = (e: Event) => { const detail = (e as CustomEvent).detail if (!detail?.message) return + /** A mode-bearing send (Ask) belongs to the home chat, which has the mode; left unclaimed, it is stored for that surface. */ + if (detail.requestMode) return e.preventDefault() setActiveTab('copilot') copilotSendMessage(detail.message, detail.fileAttachments, detail.contexts, { diff --git a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/sidebar.tsx b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/sidebar.tsx index 46a2e7b6a6f..753b54dda44 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/sidebar.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/components/sidebar/sidebar.tsx @@ -793,9 +793,12 @@ export const Sidebar = memo(function Sidebar({ label: 'Integrations', icon: Integration, href: `/workspace/${workspaceId}/integrations`, - /* Skills is a tab of this surface, not its own nav item — keep the entry - lit while the user is on it. */ - additionalActivePaths: [`/workspace/${workspaceId}/skills`], + /* Skills and Search are tabs of this surface, not their own nav items — + keep the entry lit while the user is on either. */ + additionalActivePaths: [ + `/workspace/${workspaceId}/skills`, + `/workspace/${workspaceId}/search`, + ], hidden: permissionConfig.hideIntegrationsTab, }, ].filter((item) => !item.hidden), diff --git a/apps/sim/background/knowledge-connector-member-sync.test.ts b/apps/sim/background/knowledge-connector-member-sync.test.ts new file mode 100644 index 00000000000..dbacf478487 --- /dev/null +++ b/apps/sim/background/knowledge-connector-member-sync.test.ts @@ -0,0 +1,109 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockAssertPayload, mockExecuteMemberSync, mockTask } = vi.hoisted(() => ({ + mockAssertPayload: vi.fn(), + mockExecuteMemberSync: vi.fn(), + mockTask: vi.fn((config) => config), +})) + +vi.mock('@trigger.dev/sdk', () => ({ + task: mockTask, + AbortTaskRunError: class AbortTaskRunError extends Error {}, +})) +vi.mock('@/lib/knowledge/connectors/member-queue', () => ({ + MEMBER_SYNC_TASK_ID: 'knowledge-connector-member-sync', + assertMemberSyncPayload: mockAssertPayload, +})) +vi.mock('@/lib/knowledge/connectors/member-sync-engine', () => ({ + executeMemberSync: mockExecuteMemberSync, +})) + +import { AbortTaskRunError } from '@trigger.dev/sdk' +import { + classifyMemberSyncResult, + executeMemberSyncJob, + knowledgeConnectorMemberSync, +} from '@/background/knowledge-connector-member-sync' + +const RESULT = { + docsAdded: 0, + docsUpdated: 0, + docsDeleted: 0, + docsUnchanged: 0, + docsSkipped: 0, + docsFailed: 0, + processingDispatch: { requested: 0, accepted: 0, failed: 0 }, + membersClaimed: 2, + membersCompleted: 2, + membersIncomplete: 0, + membersFailed: 0, + membersRemaining: false, + docsListed: 4, + docsHydratedOnce: 4, + observationsAdded: 4, + observationsRemoved: 0, + docsTombstoned: 0, + docsResurrected: 0, + docsPurged: 0, + credentialsAudited: 2, +} + +const PAYLOAD = { + connectorId: 'c-1', + requestId: 'r-1', + billingAttribution: { workspaceId: 'ws-1' }, + dispatchToken: 't-1', +} + +describe('knowledge connector member sync worker', () => { + beforeEach(() => { + vi.clearAllMocks() + mockAssertPayload.mockReturnValue(PAYLOAD) + mockExecuteMemberSync.mockResolvedValue(RESULT) + }) + + it('classifies outcomes from the run counters', () => { + expect(classifyMemberSyncResult(RESULT)).toBe('completed') + expect(classifyMemberSyncResult({ ...RESULT, membersFailed: 1 })).toBe('partial') + expect(classifyMemberSyncResult({ ...RESULT, docsFailed: 1 })).toBe('partial') + expect(classifyMemberSyncResult({ ...RESULT, error: 'boom' })).toBe('failed') + expect(classifyMemberSyncResult({ ...RESULT, skipReason: 'sync_in_progress' })).toBe('skipped') + }) + + it('runs the engine with the payload token and reports the outcome', async () => { + await expect(executeMemberSyncJob(PAYLOAD)).resolves.toMatchObject({ + success: true, + outcome: 'completed', + connectorId: 'c-1', + membersCompleted: 2, + }) + expect(mockExecuteMemberSync).toHaveBeenCalledWith('c-1', { + billingAttribution: PAYLOAD.billingAttribution, + dispatchToken: 't-1', + }) + }) + + it('aborts rather than retries a failed run', async () => { + mockExecuteMemberSync.mockResolvedValue({ ...RESULT, error: 'source down' }) + await expect(executeMemberSyncJob(PAYLOAD)).rejects.toBeInstanceOf(AbortTaskRunError) + }) + + it('reports a partial run without aborting, so members retry on their own ladder', async () => { + mockExecuteMemberSync.mockResolvedValue({ ...RESULT, membersFailed: 1 }) + await expect(executeMemberSyncJob(PAYLOAD)).resolves.toMatchObject({ + success: false, + outcome: 'partial', + }) + }) + + it('registers a single-attempt task on its own queue', () => { + expect(knowledgeConnectorMemberSync).toMatchObject({ + id: 'knowledge-connector-member-sync', + retry: { maxAttempts: 1 }, + queue: { name: 'connector-member-sync-queue' }, + }) + }) +}) diff --git a/apps/sim/background/knowledge-connector-member-sync.ts b/apps/sim/background/knowledge-connector-member-sync.ts new file mode 100644 index 00000000000..04cd5e1ded4 --- /dev/null +++ b/apps/sim/background/knowledge-connector-member-sync.ts @@ -0,0 +1,90 @@ +import { createLogger } from '@sim/logger' +import { AbortTaskRunError, task } from '@trigger.dev/sdk' +import { + assertMemberSyncPayload, + MEMBER_SYNC_TASK_ID, + type MemberSyncPayload, +} from '@/lib/knowledge/connectors/member-queue' +import { + executeMemberSync, + type MemberSyncResult, +} from '@/lib/knowledge/connectors/member-sync-engine' +import { MEMBER_SYNC_MAX_DURATION_SECONDS } from '@/lib/knowledge/connectors/sync-limits' + +const logger = createLogger('TriggerKnowledgeConnectorMemberSync') + +export type MemberSyncTaskOutcome = 'completed' | 'partial' | 'skipped' | 'failed' + +/** A run is partial when any member or document failed; skipped and failed mirror the content task. */ +export function classifyMemberSyncResult(result: MemberSyncResult): MemberSyncTaskOutcome { + if (result.skipReason) return 'skipped' + if (result.error) return 'failed' + if (result.membersFailed > 0 || result.docsFailed > 0 || result.processingDispatch.failed > 0) { + return 'partial' + } + return 'completed' +} + +export async function executeMemberSyncJob(payload: unknown) { + const { connectorId, requestId, billingAttribution, dispatchToken } = + assertMemberSyncPayload(payload) + + logger.info(`[${requestId}] Starting member sync: ${connectorId}`) + + try { + const result = await executeMemberSync(connectorId, { billingAttribution, dispatchToken }) + const outcome = classifyMemberSyncResult(result) + + logger.info(`[${requestId}] Member sync completed`, { + connectorId, + outcome, + membersClaimed: result.membersClaimed, + membersCompleted: result.membersCompleted, + membersIncomplete: result.membersIncomplete, + membersFailed: result.membersFailed, + membersRemaining: result.membersRemaining, + docsListed: result.docsListed, + added: result.docsAdded, + updated: result.docsUpdated, + unchanged: result.docsUnchanged, + failed: result.docsFailed, + observationsAdded: result.observationsAdded, + observationsRemoved: result.observationsRemoved, + tombstoned: result.docsTombstoned, + resurrected: result.docsResurrected, + purged: result.docsPurged, + }) + + if (outcome === 'failed') { + /** + * The engine has already written its terminal state and re-armed the + * connector's failure ladder. Retrying the task would run a second crawl + * over the same members, so fail visibly without a retry. + */ + throw new AbortTaskRunError(`Member sync failed for ${connectorId}: ${result.error}`) + } + + return { success: outcome === 'completed', outcome, connectorId, ...result } + } catch (error) { + logger.error(`[${requestId}] Member sync failed: ${connectorId}`, error) + throw error + } +} + +export const knowledgeConnectorMemberSync = task({ + id: MEMBER_SYNC_TASK_ID, + maxDuration: MEMBER_SYNC_MAX_DURATION_SECONDS, + /** Sized like the content sync: a members-mode run hydrates the same documents. */ + machine: 'large-1x', + /** + * No retries: the run re-dispatches itself while members remain due, and a + * crashed run is reclaimed by the scheduler's lease sweep, so a platform + * retry would only race the replacement. + */ + retry: { maxAttempts: 1 }, + queue: { + concurrencyLimit: 5, + name: 'connector-member-sync-queue', + }, + run: async (payload: MemberSyncPayload) => executeMemberSyncJob(payload), +}) diff --git a/apps/sim/blocks/blocks/knowledge.ts b/apps/sim/blocks/blocks/knowledge.ts index 705067951eb..6d1fea3ac36 100644 --- a/apps/sim/blocks/blocks/knowledge.ts +++ b/apps/sim/blocks/blocks/knowledge.ts @@ -164,10 +164,11 @@ export const KnowledgeBlock: BlockConfig = { title: 'Retrieval Mode', type: 'dropdown', options: [ - { label: 'Vector only', id: 'vector' }, + { label: 'Automatic', id: 'auto' }, { label: 'Hybrid (full-text + vector)', id: 'hybrid' }, + { label: 'Vector only', id: 'vector' }, ], - value: () => 'vector', + value: () => 'auto', mode: 'advanced', condition: { field: 'operation', value: 'search' }, }, @@ -539,7 +540,8 @@ export const KnowledgeBlock: BlockConfig = { tagFilters: { type: 'string', description: 'Tag filter criteria' }, searchMode: { type: 'string', - description: 'Retrieval mode: vector only (default) or hybrid (full-text + vector)', + description: + "Retrieval mode: 'hybrid' (full-text + vector) or 'vector'; omitted, the workspace's default applies", }, rerankerEnabled: { type: 'boolean', description: 'Apply Cohere reranking to search results' }, rerankerModel: { type: 'string', description: 'Cohere rerank model identifier' }, diff --git a/apps/sim/components/emails/_styles/base.tokens.test.ts b/apps/sim/components/emails/_styles/base.tokens.test.ts index 0baac25f776..9d4b794a32b 100644 --- a/apps/sim/components/emails/_styles/base.tokens.test.ts +++ b/apps/sim/components/emails/_styles/base.tokens.test.ts @@ -91,8 +91,16 @@ describe('email geometry mirrors the platform', () => { }) it('the CTA transcribes chipGeometryClass', () => { - const geometry = chipChrome.match(/chipGeometryClass = `([^`]+)`/)?.[1] - expect(geometry).toBeDefined() + // chipGeometryClass composes the unrounded geometry with the default radius, + // so the transcription reads both halves rather than one literal. + expect(chipChrome).toMatch( + /chipGeometryClass = `\$\{chipGeometryUnroundedClass\} \$\{chipRadiusClass\}`/ + ) + const unrounded = chipChrome.match(/chipGeometryUnroundedClass = `([^`]+)`/)?.[1] + const radius = chipChrome.match(/chipRadiusClass = '([^']+)'/)?.[1] + expect(unrounded).toBeDefined() + expect(radius).toBeDefined() + const geometry = `${unrounded} ${radius}` for (const token of ['h-[30px]', 'rounded-lg', 'px-2', 'text-sm']) { expect(geometry).toContain(token) } diff --git a/apps/sim/connectors/airtable/airtable.ts b/apps/sim/connectors/airtable/airtable.ts index cb955b91a21..fdddb724f41 100644 --- a/apps/sim/connectors/airtable/airtable.ts +++ b/apps/sim/connectors/airtable/airtable.ts @@ -3,7 +3,12 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { airtableConnectorMeta } from '@/connectors/airtable/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { computeContentHash, parseTagDate } from '@/connectors/utils' +import { + ConnectorListingScopeUnavailableError, + computeContentHash, + isListingScopeUnavailableError, + parseTagDate, +} from '@/connectors/utils' const logger = createLogger('AirtableConnector') @@ -143,6 +148,8 @@ function readMaxRecords(sourceConfig: Record): number { export const airtableConnector: ConnectorConfig = { ...airtableConnectorMeta, + isListingScopeUnavailableError: isListingScopeUnavailableError, + /** * Lists records from `GET /v0/{baseId}/{tableIdOrName}`. * @@ -217,7 +224,15 @@ export const airtableConnector: ConnectorConfig = { * Throwing aborts the sync before reconciliation, which is the safe * outcome — the next run restarts iteration from the beginning. */ - throw new Error(`Failed to list Airtable records: ${response.status}`) + const message = `Failed to list Airtable records: ${response.status}` + /** + * Airtable answers a base or table the caller cannot reach with 403 + * (INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND) or 404: the configured scope + * is out of this caller's reach. + */ + throw response.status === 403 || response.status === 404 + ? new ConnectorListingScopeUnavailableError(message, response.status) + : new Error(message) } const data = (await response.json()) as { diff --git a/apps/sim/connectors/airtable/meta.ts b/apps/sim/connectors/airtable/meta.ts index 3044ceca757..fd4dac57c2d 100644 --- a/apps/sim/connectors/airtable/meta.ts +++ b/apps/sim/connectors/airtable/meta.ts @@ -14,6 +14,11 @@ export const airtableConnectorMeta: ConnectorMeta = { requiredScopes: ['data.records:read', 'schema.bases:read'], }, + /** + * The listing is one configured table's records under the caller's own + * token, which reaches only the bases that member granted and can read. + */ + permissionScopedListing: { capFieldIds: ['maxRecords'] }, configFields: [ { id: 'baseSelector', diff --git a/apps/sim/connectors/asana/asana.ts b/apps/sim/connectors/asana/asana.ts index 576921e2882..f6c3758a61a 100644 --- a/apps/sim/connectors/asana/asana.ts +++ b/apps/sim/connectors/asana/asana.ts @@ -314,6 +314,9 @@ async function listWorkspaceProjects( export const asanaConnector: ConnectorConfig = { ...asanaConnectorMeta, + isListingScopeUnavailableError: (error) => + error instanceof AsanaApiError && (error.status === 404 || error.status === 403), + listDocuments: async ( accessToken: string, sourceConfig: Record, diff --git a/apps/sim/connectors/asana/meta.ts b/apps/sim/connectors/asana/meta.ts index 075b14e0f2a..0d146487c82 100644 --- a/apps/sim/connectors/asana/meta.ts +++ b/apps/sim/connectors/asana/meta.ts @@ -10,6 +10,7 @@ export const asanaConnectorMeta: ConnectorMeta = { auth: { mode: 'oauth', provider: 'asana', requiredScopes: ['default'] }, + permissionScopedListing: { capFieldIds: ['maxTasks'] }, configFields: [ { id: 'workspaceSelector', diff --git a/apps/sim/connectors/bitbucket/bitbucket.test.ts b/apps/sim/connectors/bitbucket/bitbucket.test.ts index 8b184f8a1e8..e36b329a4f2 100644 --- a/apps/sim/connectors/bitbucket/bitbucket.test.ts +++ b/apps/sim/connectors/bitbucket/bitbucket.test.ts @@ -490,6 +490,21 @@ describe('bitbucket maxItems cap', () => { expect(syncContext.listingCapped).toBe(true) }) + it('reads a zero cap, which members mode writes, as unlimited', async () => { + mockApi([[/\/src\//, () => jsonResponse({ values: [fileEntry('a.md'), fileEntry('b.md')] })]]) + + const syncContext: Record = {} + const result = await bitbucketConnector.listDocuments( + ACCESS_TOKEN, + { ...CONFIG, maxItems: 0 }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(2) + expect(syncContext.listingCapped).toBeUndefined() + }) + it('leaves the listing reconcilable when no cap is configured', async () => { mockApi([[/\/src\//, () => jsonResponse({ values: [fileEntry('a.md'), fileEntry('b.md')] })]]) @@ -533,6 +548,42 @@ describe('bitbucket maxItems cap', () => { }) }) +describe('bitbucket listing scope', () => { + it.each([403, 404])( + 'reports a repository the token cannot reach (%i) as an unavailable listing scope', + async (status) => { + mockApi([[/\/repositories\/acme\/widgets$/, () => jsonResponse({ type: 'error' }, status)]]) + + const error = await bitbucketConnector + .listDocuments(ACCESS_TOKEN, CONFIG, undefined, {}) + .catch((caught: unknown) => caught) + + expect(bitbucketConnector.isListingScopeUnavailableError?.(error)).toBe(true) + } + ) + + it('reports a pull request listing the token cannot reach as an unavailable listing scope', async () => { + mockApi([[/\/pullrequests/, () => jsonResponse({ type: 'error' }, 403)]]) + + const error = await bitbucketConnector + .listDocuments(ACCESS_TOKEN, PR_CONFIG, undefined, {}) + .catch((caught: unknown) => caught) + + expect(bitbucketConnector.isListingScopeUnavailableError?.(error)).toBe(true) + }) + + it('keeps any other repository failure retryable', async () => { + mockApi([[/\/repositories\/acme\/widgets$/, () => jsonResponse({ type: 'error' }, 500)]]) + + const error = await bitbucketConnector + .listDocuments(ACCESS_TOKEN, CONFIG, undefined, {}) + .catch((caught: unknown) => caught) + + expect(error).toBeInstanceOf(Error) + expect(bitbucketConnector.isListingScopeUnavailableError?.(error)).toBe(false) + }) +}) + describe('bitbucket pull request listing', () => { it('builds the documented collection query', async () => { mockApi([[/\/pullrequests/, () => jsonResponse({ values: [pullRequestFixture(7)] })]]) @@ -844,13 +895,13 @@ describe('bitbucket validateConfig', () => { ).toEqual({ valid: true }) }) - it('rejects a non-positive maxItems before spending a request', async () => { + it('rejects a negative maxItems before spending a request', async () => { mockApi([]) expect( await bitbucketConnector.validateConfig(ACCESS_TOKEN, { ...CONFIG, - maxItems: '0', + maxItems: '-1', }) ).toEqual({ valid: false, error: 'Max items must be a positive integer' }) }) diff --git a/apps/sim/connectors/bitbucket/bitbucket.ts b/apps/sim/connectors/bitbucket/bitbucket.ts index 484fbdcaf47..b0d5239ee9a 100644 --- a/apps/sim/connectors/bitbucket/bitbucket.ts +++ b/apps/sim/connectors/bitbucket/bitbucket.ts @@ -5,6 +5,8 @@ import { bitbucketConnectorMeta } from '@/connectors/bitbucket/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { CONNECTOR_MAX_FILE_BYTES, + ConnectorListingScopeUnavailableError, + isListingScopeUnavailableError, markSkipped, parseTagDate, readBodyWithLimit, @@ -223,12 +225,15 @@ function readSlug(value: unknown): string { return typeof value === 'string' ? value.trim() : '' } -/** Reads an optional positive, finite integer document cap. */ +/** + * Reads the optional document cap: a positive, finite integer, or 0 for + * unlimited, which is also how a members-mode sync clears the cap. + */ function readMaxItems(value: unknown): number { if (value === undefined || value === null) return 0 if (typeof value === 'string' && !value.trim()) return 0 const parsed = Number(value) - if (!Number.isSafeInteger(parsed) || parsed <= 0) { + if (!Number.isSafeInteger(parsed) || parsed < 0) { throw new Error('Max items must be a positive integer') } return parsed @@ -520,6 +525,18 @@ function pullRequestToDocument( * Fetches the repository record, used to resolve the canonical full name, the web * UI base URL, and the default branch — and to confirm access during validation. */ +/** + * The error a request against the configured repository throws: scope-unavailable + * when Bitbucket says the repository does not exist for this caller (404) or + * refuses them (403), a plain error for anything else. + */ +function repositoryRequestError(message: string, status: number): Error { + const described = `${message}: ${status}` + return status === 403 || status === 404 + ? new ConnectorListingScopeUnavailableError(described, status) + : new Error(described) +} + async function fetchRepository( workspaceSlug: string, repoSlug: string, @@ -553,8 +570,9 @@ async function resolveRepository( const response = await fetchRepository(workspaceSlug, repoSlug, accessToken) if (!response.ok) { - throw new Error( - `Cannot access Bitbucket repository ${workspaceSlug}/${repoSlug}: ${response.status}` + throw repositoryRequestError( + `Cannot access Bitbucket repository ${workspaceSlug}/${repoSlug}`, + response.status ) } @@ -808,6 +826,8 @@ function applyMaxItemsCap( export const bitbucketConnector: ConnectorConfig = { ...bitbucketConnectorMeta, + isListingScopeUnavailableError: isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, @@ -1097,7 +1117,7 @@ export const bitbucketConnector: ConnectorConfig = { status: response.status, error: errorText.slice(0, 500), }) - throw new Error(`Failed to list Bitbucket pull requests: ${response.status}`) + throw repositoryRequestError('Failed to list Bitbucket pull requests', response.status) } const page = parsePagedResponse( diff --git a/apps/sim/connectors/bitbucket/meta.ts b/apps/sim/connectors/bitbucket/meta.ts index 6f599b8f314..d92c1498b91 100644 --- a/apps/sim/connectors/bitbucket/meta.ts +++ b/apps/sim/connectors/bitbucket/meta.ts @@ -34,6 +34,12 @@ export const bitbucketConnectorMeta: ConnectorMeta = { requiredScopes: ['pullrequest'], }, + /** + * The listing is one configured repository's files and pull requests: a + * member with read access to the repository lists all of them, one without + * lists nothing. + */ + permissionScopedListing: { capFieldIds: ['maxItems'] }, configFields: [ { id: 'workspaceSelector', diff --git a/apps/sim/connectors/box/box.test.ts b/apps/sim/connectors/box/box.test.ts new file mode 100644 index 00000000000..1ab7d1b4d6f --- /dev/null +++ b/apps/sim/connectors/box/box.test.ts @@ -0,0 +1,78 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() })) + +vi.mock('@/lib/knowledge/documents/utils', () => ({ + fetchWithRetry: mockFetchWithRetry, + VALIDATE_RETRY_OPTIONS: {}, +})) +vi.mock('@/components/icons', () => ({ BoxCompanyIcon: () => null })) + +import { boxConnector } from '@/connectors/box/box' +import { PER_MEMBER_LISTING_CONTEXT } from '@/connectors/utils' + +interface FolderReply { + status?: number + entries?: unknown[] +} + +/** Routes `GET /folders/:id/items` by folder id; unknown folders answer 404. */ +function mockFolders(folders: Record) { + mockFetchWithRetry.mockImplementation(async (url: string) => { + const folderId = /\/folders\/([^/]+)\/items/.exec(url)?.[1] ?? '' + const reply = folders[folderId] ?? { status: 404 } + const status = reply.status ?? 200 + const body = { entries: reply.entries ?? [], next_marker: null } + return { + ok: status >= 200 && status < 300, + status, + json: async () => body, + text: async () => JSON.stringify(body), + } as unknown as Response + }) +} + +const FILE = { type: 'file', id: 'f1', name: 'notes.txt', extension: 'txt', size: 10 } +const SUBFOLDER = { type: 'folder', id: 'sub', name: 'Private' } + +beforeEach(() => { + vi.clearAllMocks() +}) + +describe('box listDocuments with a subfolder the caller cannot reach', () => { + it('caps the listing under a shared credential so nothing is reconciled as deleted', async () => { + mockFolders({ '0': { entries: [FILE, SUBFOLDER] }, sub: { status: 403 } }) + const syncContext: Record = {} + + const result = await boxConnector.listDocuments('token', {}, undefined, syncContext) + + expect(result.documents.map((doc) => doc.externalId)).toEqual(['f1']) + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBe(true) + }) + + it("lists completely under a member's own token so their access to it is withdrawn", async () => { + mockFolders({ '0': { entries: [FILE, SUBFOLDER] }, sub: { status: 403 } }) + const syncContext: Record = { ...PER_MEMBER_LISTING_CONTEXT } + + const result = await boxConnector.listDocuments('token', {}, undefined, syncContext) + + expect(result.documents.map((doc) => doc.externalId)).toEqual(['f1']) + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('reports the configured root being unreachable as the scope being unavailable', async () => { + mockFolders({ '42': { status: 403 } }) + const syncContext: Record = { ...PER_MEMBER_LISTING_CONTEXT } + + const error = await boxConnector + .listDocuments('token', { folderId: '42' }, undefined, syncContext) + .catch((caught: unknown) => caught) + + expect(boxConnector.isListingScopeUnavailableError?.(error)).toBe(true) + }) +}) diff --git a/apps/sim/connectors/box/box.ts b/apps/sim/connectors/box/box.ts index aa5bcae9a82..3f36323905f 100644 --- a/apps/sim/connectors/box/box.ts +++ b/apps/sim/connectors/box/box.ts @@ -7,7 +7,10 @@ import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/ import { CONNECTOR_MAX_FILE_BYTES, ConnectorFileTooLargeError, + ConnectorListingScopeUnavailableError, htmlToPlainText, + isListingScopeUnavailableError, + isPerMemberListing, isSkippedDocument, markSkipped, parseTagDate, @@ -383,7 +386,7 @@ async function fetchExtractedText( /** * Lists one page of a folder. A folder the credential can no longer read is * reported rather than thrown, so one inaccessible subtree does not abort the - * whole listing — the caller flags the listing as capped instead. + * whole listing — the caller decides whether that caps the listing. */ async function listFolderPage( accessToken: string, @@ -426,6 +429,8 @@ async function listFolderPage( export const boxConnector: ConnectorConfig = { ...boxConnectorMeta, + isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, @@ -453,8 +458,9 @@ export const boxConnector: ConnectorConfig = { * reporting a successful sync that indexed nothing. */ if (!page && position.folderId === rootFolderId) { - throw new Error( - `Box denied access to folder ${rootFolderId}. Reconnect the Box account or choose another folder.` + throw new ConnectorListingScopeUnavailableError( + `Box denied access to folder ${rootFolderId}. Reconnect the Box account or choose another folder.`, + 403 ) } @@ -466,10 +472,13 @@ export const boxConnector: ConnectorConfig = { files.push(item) } } - } else if (syncContext) { + } else if (syncContext && !isPerMemberListing(syncContext)) { /** * A folder was skipped, so documents that still exist in Box are absent from - * this listing. Without this flag the engine would reconcile them as deleted. + * this listing. Under a shared credential the engine would otherwise + * reconcile them as deleted; under a member's own token the folder is + * simply not shared with that member, so their listing stays complete and + * their access to its files is withdrawn. */ syncContext.listingCapped = true } diff --git a/apps/sim/connectors/box/meta.ts b/apps/sim/connectors/box/meta.ts index a8c2643ef30..9670b0575f0 100644 --- a/apps/sim/connectors/box/meta.ts +++ b/apps/sim/connectors/box/meta.ts @@ -14,6 +14,7 @@ export const boxConnectorMeta: ConnectorMeta = { requiredScopes: ['root_readwrite'], }, + permissionScopedListing: { capFieldIds: ['maxFiles'] }, configFields: [ { id: 'folderId', diff --git a/apps/sim/connectors/clickup/clickup.ts b/apps/sim/connectors/clickup/clickup.ts index 7d42855face..87a9ac3bce6 100644 --- a/apps/sim/connectors/clickup/clickup.ts +++ b/apps/sim/connectors/clickup/clickup.ts @@ -4,7 +4,11 @@ import { isRecordLike } from '@sim/utils/object' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { clickupConnectorMeta } from '@/connectors/clickup/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { parseTagDate } from '@/connectors/utils' +import { + isListingScopeUnavailableError, + listingRequestError, + parseTagDate, +} from '@/connectors/utils' import { clickupAuthorizationHeader, extractClickUpErrorMessage } from '@/tools/clickup/shared' const logger = createLogger('ClickUpConnector') @@ -159,6 +163,8 @@ function getRequiredWorkspaceId(sourceConfig: Record): string { export const clickupConnector: ConnectorConfig = { ...clickupConnectorMeta, + isListingScopeUnavailableError: isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, @@ -181,7 +187,11 @@ export const clickupConnector: ConnectorConfig = { if (!response.ok) { const errorText = await response.text() logger.error('Failed to list ClickUp Docs', { status: response.status, error: errorText }) - throw new Error(`Failed to list ClickUp Docs: ${response.status}`) + throw listingRequestError( + 'Failed to list ClickUp Docs', + response.status, + response.status === 404 || (response.status === 401 && /OAUTH_02[37]/.test(errorText)) + ) } const data = (await response.json()) as Record diff --git a/apps/sim/connectors/clickup/meta.ts b/apps/sim/connectors/clickup/meta.ts index 5879b5cf2ab..76004caeefd 100644 --- a/apps/sim/connectors/clickup/meta.ts +++ b/apps/sim/connectors/clickup/meta.ts @@ -13,6 +13,7 @@ export const clickupConnectorMeta: ConnectorMeta = { provider: 'clickup', }, + permissionScopedListing: { capFieldIds: ['maxDocs'] }, configFields: [ { id: 'workspaceSelector', diff --git a/apps/sim/connectors/confluence/confluence.test.ts b/apps/sim/connectors/confluence/confluence.test.ts index 985b635b56f..383a8ca8164 100644 --- a/apps/sim/connectors/confluence/confluence.test.ts +++ b/apps/sim/connectors/confluence/confluence.test.ts @@ -1,8 +1,14 @@ /** * @vitest-environment node */ -import { describe, expect, it } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { + AtlassianSiteNotAccessibleError, + AtlassianSiteNotMatchedError, +} from '@/lib/atlassian/discovery' +import { + buildLastModifiedClause, + confluenceConnector, escapeCql, extractCursor, isCurrentContent, @@ -37,6 +43,45 @@ describe('escapeCql', () => { }) }) +describe('buildLastModifiedClause', () => { + const now = new Date('2026-09-01T12:00:00Z') + + it.concurrent('rounds the watermark up to whole minutes relative to the server clock', () => { + expect(buildLastModifiedClause(new Date('2026-09-01T11:30:30Z'), now)).toBe( + 'lastModified >= now("-30m")' + ) + }) + + it.concurrent('never asks for less than a minute', () => { + expect(buildLastModifiedClause(now, now)).toBe('lastModified >= now("-1m")') + expect(buildLastModifiedClause(new Date(now.getTime() + 60_000), now)).toBe( + 'lastModified >= now("-1m")' + ) + }) +}) + +describe('confluence listing scope classification', () => { + it.concurrent('treats a token that reaches no Atlassian site as not on the site', () => { + expect( + confluenceConnector.isListingScopeUnavailableError?.( + new AtlassianSiteNotAccessibleError('none') + ) + ).toBe(true) + }) + + it.concurrent('treats a token that reaches only other Atlassian sites the same way', () => { + expect( + confluenceConnector.isListingScopeUnavailableError?.( + new AtlassianSiteNotMatchedError('elsewhere') + ) + ).toBe(true) + }) + + it.concurrent('leaves other failures for the sync engines to retry', () => { + expect(confluenceConnector.isListingScopeUnavailableError?.(new Error('boom'))).toBe(false) + }) +}) + describe('isCurrentContent', () => { it.concurrent('keeps current content', () => { expect(isCurrentContent({ id: '1', status: 'current' })).toBe(true) @@ -363,3 +408,66 @@ describe('preserveConfluenceCallouts', () => { expect(result).toContain('[WARNING] Do NOT use this form for: GitLab') }) }) + +describe('confluence incremental CQL listing', () => { + const fetchMock = + vi.fn<(input: string | URL | Request, init?: RequestInit) => Promise>() + + function jsonResponse(body: unknown): Response { + return new Response(JSON.stringify(body), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + } + + function cqlOfCall(index: number): string | null { + return new URL(String(fetchMock.mock.calls[index][0])).searchParams.get('cql') + } + + beforeEach(() => { + vi.useFakeTimers() + fetchMock.mockReset() + vi.stubGlobal('fetch', fetchMock) + }) + + afterEach(() => { + vi.unstubAllGlobals() + vi.useRealTimers() + }) + + it('keeps one lastModified clause across pages that straddle a minute boundary', async () => { + const lastSyncAt = new Date('2026-09-01T11:30:00Z') + const config = { domain: 'example.atlassian.net', spaceKey: 'ENG' } + const syncContext: Record = { cloudId: 'cloud-1' } + fetchMock + .mockResolvedValueOnce( + jsonResponse({ + results: [], + _links: { next: '/wiki/rest/api/content/search?cursor=page-2&cql=ignored' }, + }) + ) + .mockResolvedValueOnce(jsonResponse({ results: [] })) + + vi.setSystemTime(new Date('2026-09-01T12:00:59Z')) + const first = await confluenceConnector.listDocuments( + 'token', + config, + undefined, + syncContext, + lastSyncAt + ) + expect(first.nextCursor).toBe('page-2') + + vi.setSystemTime(new Date('2026-09-01T12:01:01Z')) + await confluenceConnector.listDocuments( + 'token', + config, + first.nextCursor, + syncContext, + lastSyncAt + ) + + expect(cqlOfCall(0)).toContain('lastModified >= now("-31m")') + expect(cqlOfCall(1)).toBe(cqlOfCall(0)) + }) +}) diff --git a/apps/sim/connectors/confluence/confluence.ts b/apps/sim/connectors/confluence/confluence.ts index 381254d79f1..6ad3f97e199 100644 --- a/apps/sim/connectors/confluence/confluence.ts +++ b/apps/sim/connectors/confluence/confluence.ts @@ -1,6 +1,10 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import * as cheerio from 'cheerio' +import { + AtlassianSiteNotAccessibleError, + AtlassianSiteNotMatchedError, +} from '@/lib/atlassian/discovery' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { confluenceConnectorMeta } from '@/connectors/confluence/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' @@ -9,6 +13,18 @@ import { getConfluenceCloudId, normalizeConfluenceDomainHost } from '@/tools/con const logger = createLogger('ConfluenceConnector') +/** + * The configured space does not exist for the caller. Confluence answers a + * space lookup with an empty result rather than a 403 when the caller cannot + * see the space, so this is also what a member without access observes. + */ +export class ConfluenceSpaceNotFoundError extends Error { + constructor(readonly spaceKey: string) { + super(`Space "${spaceKey}" not found`) + this.name = 'ConfluenceSpaceNotFoundError' + } +} + /** Label prefixes for Confluence's built-in Info/Note/Warning/Tip macros, by their rendered CSS suffix. */ const CALLOUT_LABELS: Record = { information: '[INFO]', @@ -289,7 +305,8 @@ export const confluenceConnector: ConnectorConfig = { accessToken: string, sourceConfig: Record, cursor?: string, - syncContext?: Record + syncContext?: Record, + lastSyncAt?: Date ): Promise => { const domain = normalizeConfluenceDomainHost(sourceConfig.domain as string) const spaceKeys = parseMultiValue(sourceConfig.spaceKey) @@ -308,11 +325,13 @@ export const confluenceConnector: ConnectorConfig = { } /** - * Route through CQL when a label filter is set or when multiple spaces are - * selected — the v2 `/spaces/{spaceId}/pages` endpoint is single-space only, - * but CQL natively supports `space in (...)`. + * Route through CQL when a label filter is set, when multiple spaces are + * selected, or when only recently modified content is wanted — the v2 + * `/spaces/{spaceId}/pages` endpoint is single-space only and cannot filter + * by modification time, but CQL natively supports `space in (...)` and + * `lastModified`. */ - if (labelFilter.trim() || spaceKeys.length > 1) { + if (labelFilter.trim() || spaceKeys.length > 1 || lastSyncAt) { return listDocumentsViaCql( cloudId, accessToken, @@ -322,7 +341,8 @@ export const confluenceConnector: ConnectorConfig = { labelFilter, maxPages, cursor, - syncContext + syncContext, + lastSyncAt ) } @@ -489,6 +509,16 @@ export const confluenceConnector: ConnectorConfig = { return result }, + + /** + * A member who is not on the Atlassian site — their token reaches no site, + * or only sites other than the configured one — or who cannot see the + * configured space, lists nothing: a complete listing of nothing, not an error. + */ + isListingScopeUnavailableError: (error) => + error instanceof ConfluenceSpaceNotFoundError || + error instanceof AtlassianSiteNotAccessibleError || + error instanceof AtlassianSiteNotMatchedError, } /** @@ -657,6 +687,34 @@ async function listAllContentTypes( return results } +/** + * The CQL clause selecting content modified since a watermark. CQL's `now()` + * takes a relative offset and evaluates on the server, which sidesteps the + * timezone the endpoint would otherwise assume for an absolute timestamp; the + * offset rounds up to the next whole minute so nothing at the edge is missed. + */ +export function buildLastModifiedClause(lastSyncAt: Date, now: Date): string { + const minutes = Math.max(1, Math.ceil((now.getTime() - lastSyncAt.getTime()) / 60_000)) + return `lastModified >= now("-${minutes}m")` +} + +/** + * The `lastModified` clause every page of one listing shares. The clause is a + * window relative to the server clock, so recomputing it on a later page that + * crosses a minute boundary would pair the cursor `_links.next` issued with a + * query it was not issued for; the first page fixes it for the run. + */ +export function resolveLastModifiedClause( + lastSyncAt: Date, + syncContext: Record | undefined +): string { + const fixed = syncContext?.cqlLastModifiedClause + if (typeof fixed === 'string') return fixed + const clause = buildLastModifiedClause(lastSyncAt, new Date()) + if (syncContext) syncContext.cqlLastModifiedClause = clause + return clause +} + /** * Page size for CQL search. The endpoint defaults to 25 and documents no hard * maximum, so this stays conservatively below the fixed system limits it warns @@ -676,7 +734,8 @@ async function listDocumentsViaCql( labelFilter: string, maxPages: number, cursor?: string, - syncContext?: Record + syncContext?: Record, + lastSyncAt?: Date ): Promise { const labels = labelFilter .split(',') @@ -707,6 +766,8 @@ async function listDocumentsViaCql( cql += ` AND label in (${labelList})` } + if (lastSyncAt) cql += ` AND ${resolveLastModifiedClause(lastSyncAt, syncContext)}` + const fetchedSoFar = (syncContext?.totalDocsFetched as number) ?? 0 const remaining = maxPages > 0 ? maxPages - fetchedSoFar : Number.POSITIVE_INFINITY @@ -815,7 +876,7 @@ async function resolveSpaceId( const results = data.results || [] if (results.length === 0) { - throw new Error(`Space "${spaceKey}" not found`) + throw new ConfluenceSpaceNotFoundError(spaceKey) } return String(results[0].id) diff --git a/apps/sim/connectors/confluence/meta.ts b/apps/sim/connectors/confluence/meta.ts index 83c3aa98bee..ed114e242ea 100644 --- a/apps/sim/connectors/confluence/meta.ts +++ b/apps/sim/connectors/confluence/meta.ts @@ -31,6 +31,9 @@ export const confluenceConnectorMeta: ConnectorMeta = { */ rehydrateOnFullSync: true, + /** CQL search under a member's token returns only content that member may view. */ + permissionScopedListing: { capFieldIds: ['maxPages'] }, + configFields: [ { id: 'domain', diff --git a/apps/sim/connectors/docusign/meta.ts b/apps/sim/connectors/docusign/meta.ts index 08a67c77e47..86e59da2770 100644 --- a/apps/sim/connectors/docusign/meta.ts +++ b/apps/sim/connectors/docusign/meta.ts @@ -16,6 +16,7 @@ export const docusignConnectorMeta: ConnectorMeta = { supportsIncrementalSync: true, + permissionScopedListing: { capFieldIds: ['maxEnvelopes'] }, configFields: [ { id: 'lookback', diff --git a/apps/sim/connectors/dropbox/dropbox.ts b/apps/sim/connectors/dropbox/dropbox.ts index 5e44cfb9edd..fbeb6c5d4a7 100644 --- a/apps/sim/connectors/dropbox/dropbox.ts +++ b/apps/sim/connectors/dropbox/dropbox.ts @@ -7,7 +7,9 @@ import { CONNECTOR_MAX_FILE_BYTES, ConnectorFileTooLargeError, htmlToPlainText, + isListingScopeUnavailableError, isSkippedDocument, + listingRequestError, markSkipped, parseTagDate, readBodyWithLimit, @@ -170,6 +172,8 @@ function fileToStub(entry: DropboxFileMetadata): ExternalDocument { export const dropboxConnector: ConnectorConfig = { ...dropboxConnectorMeta, + isListingScopeUnavailableError: isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, @@ -227,7 +231,15 @@ export const dropboxConnector: ConnectorConfig = { status: response.status, error: errorText, }) - throw new Error(`Failed to list Dropbox folder: ${response.status}`) + /** + * Dropbox answers every endpoint-specific failure with 409; only + * path/not_found means the caller cannot reach the folder. + */ + throw listingRequestError( + 'Failed to list Dropbox folder', + response.status, + response.status === 409 && /path\/not_found/.test(errorText) + ) } data = await response.json() diff --git a/apps/sim/connectors/dropbox/meta.ts b/apps/sim/connectors/dropbox/meta.ts index 9e294b6fe7f..655935dbc25 100644 --- a/apps/sim/connectors/dropbox/meta.ts +++ b/apps/sim/connectors/dropbox/meta.ts @@ -14,6 +14,7 @@ export const dropboxConnectorMeta: ConnectorMeta = { requiredScopes: ['files.metadata.read', 'files.content.read'], }, + permissionScopedListing: { capFieldIds: ['maxFiles'] }, configFields: [ { id: 'folderPath', diff --git a/apps/sim/connectors/gmail/gmail.test.ts b/apps/sim/connectors/gmail/gmail.test.ts new file mode 100644 index 00000000000..7f4922325dc --- /dev/null +++ b/apps/sim/connectors/gmail/gmail.test.ts @@ -0,0 +1,117 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() })) + +vi.mock('@/lib/knowledge/documents/utils', () => ({ + fetchWithRetry: mockFetchWithRetry, + VALIDATE_RETRY_OPTIONS: {}, +})) +vi.mock('@/components/icons', () => ({ GmailIcon: () => null })) + +import { gmailConnector } from '@/connectors/gmail/gmail' +import { DEFAULT_MAX_THREADS } from '@/connectors/gmail/meta' + +function threads(count: number, prefix: string) { + return Array.from({ length: count }, (_, i) => ({ id: `${prefix}-${i}`, historyId: '1' })) +} + +/** Queues thread-list pages in order; each call records the requested URL. */ +function mockPages(pages: { threads: unknown[]; nextPageToken?: string }[]) { + const urls: string[] = [] + let call = 0 + mockFetchWithRetry.mockImplementation(async (url: string) => { + urls.push(url) + const page = pages[call++] ?? { threads: [] } + return { + ok: true, + status: 200, + json: async () => page, + text: async () => JSON.stringify(page), + } as unknown as Response + }) + return urls +} + +beforeEach(() => { + vi.clearAllMocks() +}) + +describe('gmail listDocuments with maxThreads 0 (unlimited, a per-member sync)', () => { + it('pages past a full page and never marks the listing capped', async () => { + const urls = mockPages([ + { threads: threads(100, 'a'), nextPageToken: 'page-2' }, + { threads: threads(50, 'b') }, + ]) + const syncContext: Record = {} + + const first = await gmailConnector.listDocuments( + 'token', + { maxThreads: 0 }, + undefined, + syncContext + ) + expect(first.documents).toHaveLength(100) + expect(first.hasMore).toBe(true) + expect(first.nextCursor).toBe('page-2') + expect(syncContext.listingCapped).toBeUndefined() + + const second = await gmailConnector.listDocuments( + 'token', + { maxThreads: 0 }, + first.nextCursor, + syncContext + ) + expect(second.documents).toHaveLength(50) + expect(second.hasMore).toBe(false) + expect(syncContext.totalThreadsFetched).toBe(150) + expect(syncContext.listingCapped).toBeUndefined() + expect(urls[1]).toContain('pageToken=page-2') + expect(urls[1]).toContain('maxResults=100') + }) + + it('still stops and flags a cap that truncates a longer listing', async () => { + mockPages([{ threads: threads(100, 'a'), nextPageToken: 'page-2' }]) + const syncContext: Record = {} + + const result = await gmailConnector.listDocuments( + 'token', + { maxThreads: 100 }, + undefined, + syncContext + ) + expect(result.hasMore).toBe(false) + expect(result.nextCursor).toBeUndefined() + expect(syncContext.listingCapped).toBe(true) + }) +}) + +describe('gmail listDocuments with a blank maxThreads', () => { + it.each([null, '', ' '])('keeps the default cap for %j', async (maxThreads) => { + mockPages([]) + const syncContext: Record = { totalThreadsFetched: DEFAULT_MAX_THREADS } + + const result = await gmailConnector.listDocuments( + 'token', + { maxThreads }, + undefined, + syncContext + ) + + expect(result.hasMore).toBe(false) + expect(mockFetchWithRetry).not.toHaveBeenCalled() + }) +}) + +describe('gmail validateConfig maxThreads', () => { + it('refuses what the sync parser would refuse, before any request', async () => { + for (const maxThreads of ['1.5', 'abc', '-1']) { + const result = await gmailConnector.validateConfig('token', { maxThreads }) + expect(result.valid).toBe(false) + expect(result.error).toBe('Max threads must be a non-negative whole number') + } + expect(mockFetchWithRetry).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/connectors/gmail/gmail.ts b/apps/sim/connectors/gmail/gmail.ts index 4e604f1115f..b25f6243a8c 100644 --- a/apps/sim/connectors/gmail/gmail.ts +++ b/apps/sim/connectors/gmail/gmail.ts @@ -3,7 +3,13 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { DEFAULT_MAX_THREADS, gmailConnectorMeta } from '@/connectors/gmail/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { htmlToPlainText, joinTagArray, parseMultiValue, parseTagDate } from '@/connectors/utils' +import { + htmlToPlainText, + joinTagArray, + parseDefaultedUnlimitedSafeInteger, + parseMultiValue, + parseTagDate, +} from '@/connectors/utils' const logger = createLogger('GmailConnector') @@ -446,17 +452,20 @@ export const gmailConnector: ConnectorConfig = { labelIndex = resolved } const searchQuery = buildSearchQuery(sourceConfig, labelIndex) - const maxThreads = sourceConfig.maxThreads - ? Number(sourceConfig.maxThreads) - : DEFAULT_MAX_THREADS + /** A blank field keeps the default cap; an explicit 0 (a per-member sync) means unlimited. */ + const maxThreads = parseDefaultedUnlimitedSafeInteger( + sourceConfig.maxThreads, + DEFAULT_MAX_THREADS, + 'maxThreads must be a non-negative integer' + ) const totalFetched = (syncContext?.totalThreadsFetched as number) ?? 0 - if (totalFetched >= maxThreads) { + if (maxThreads > 0 && totalFetched >= maxThreads) { return { documents: [], hasMore: false } } - const remaining = maxThreads - totalFetched - const pageSize = Math.min(THREADS_PER_PAGE, remaining) + const pageSize = + maxThreads > 0 ? Math.min(THREADS_PER_PAGE, maxThreads - totalFetched) : THREADS_PER_PAGE const queryParams = new URLSearchParams({ maxResults: String(pageSize), @@ -501,7 +510,7 @@ export const gmailConnector: ConnectorConfig = { const newTotal = totalFetched + documents.length if (syncContext) syncContext.totalThreadsFetched = newTotal - const hitLimit = newTotal >= maxThreads + const hitLimit = maxThreads > 0 && newTotal >= maxThreads /** * Only a cap that actually truncates a longer listing blocks deletion @@ -556,10 +565,15 @@ export const gmailConnector: ConnectorConfig = { accessToken: string, sourceConfig: Record ): Promise<{ valid: boolean; error?: string }> => { - const maxThreads = sourceConfig.maxThreads as string | undefined - - if (maxThreads && (Number.isNaN(Number(maxThreads)) || Number(maxThreads) <= 0)) { - return { valid: false, error: 'Max threads must be a positive number' } + /** The same parser the sync uses, so a value that saves is a value that syncs. */ + try { + parseDefaultedUnlimitedSafeInteger( + sourceConfig.maxThreads, + DEFAULT_MAX_THREADS, + 'Max threads must be a non-negative whole number' + ) + } catch (error) { + return { valid: false, error: getErrorMessage(error) } } try { diff --git a/apps/sim/connectors/gmail/meta.ts b/apps/sim/connectors/gmail/meta.ts index a5f91fcab23..bdeb1a6371b 100644 --- a/apps/sim/connectors/gmail/meta.ts +++ b/apps/sim/connectors/gmail/meta.ts @@ -16,6 +16,7 @@ export const gmailConnectorMeta: ConnectorMeta = { requiredScopes: ['https://www.googleapis.com/auth/gmail.modify'], }, + permissionScopedListing: { capFieldIds: ['maxThreads'] }, configFields: [ { id: 'labelSelector', diff --git a/apps/sim/connectors/google-calendar/google-calendar.test.ts b/apps/sim/connectors/google-calendar/google-calendar.test.ts index 00964259747..dee0f6b747a 100644 --- a/apps/sim/connectors/google-calendar/google-calendar.test.ts +++ b/apps/sim/connectors/google-calendar/google-calendar.test.ts @@ -4,6 +4,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { googleCalendarConnector } from '@/connectors/google-calendar/google-calendar' import { googleCalendarConnectorMeta } from '@/connectors/google-calendar/meta' +import { PER_MEMBER_LISTING_CONTEXT } from '@/connectors/utils' const ORGANIZER_EMAIL = 'organizer@example.com' const ATTENDEE_EMAIL = 'attendee@example.com' @@ -58,6 +59,81 @@ async function listOne(sourceConfig: Record) { return result.documents[0] } +describe('google-calendar listDocuments with a calendar the caller cannot reach', () => { + function mockCalendars(unreachable: string) { + fetchMock.mockImplementation(async (input) => { + const url = String(input) + if (url.includes(`/calendars/${unreachable}/events?`)) { + return jsonResponse({ error: { code: 404 } }, 404) + } + if (url.includes('/events?')) return jsonResponse({ items: [EVENT] }) + throw new Error(`Unexpected fetch: ${url}`) + }) + } + + it("skips only the unreachable calendar under a member's own token", async () => { + mockCalendars('alpha') + const sourceConfig = { calendarId: 'alpha,beta' } + const syncContext: Record = { ...PER_MEMBER_LISTING_CONTEXT } + + const first = await googleCalendarConnector.listDocuments( + 'token', + sourceConfig, + undefined, + syncContext + ) + expect(first.documents).toHaveLength(0) + expect(first.hasMore).toBe(true) + expect(JSON.parse(first.nextCursor ?? '{}')).toEqual({ calendarIndex: 1 }) + + const second = await googleCalendarConnector.listDocuments( + 'token', + sourceConfig, + first.nextCursor, + syncContext + ) + expect(second.documents).toHaveLength(1) + expect(second.hasMore).toBe(false) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('ends the listing when the unreachable calendar is the last one', async () => { + mockCalendars('beta') + const sourceConfig = { calendarId: 'alpha,beta' } + const syncContext: Record = { ...PER_MEMBER_LISTING_CONTEXT } + + const first = await googleCalendarConnector.listDocuments( + 'token', + sourceConfig, + undefined, + syncContext + ) + const second = await googleCalendarConnector.listDocuments( + 'token', + sourceConfig, + first.nextCursor, + syncContext + ) + expect(second.documents).toHaveLength(0) + expect(second.hasMore).toBe(false) + }) + + it('reports a sole unreachable calendar as the whole scope being unavailable', async () => { + mockCalendars('alpha') + const error = await googleCalendarConnector + .listDocuments('token', { calendarId: 'alpha' }, undefined, { ...PER_MEMBER_LISTING_CONTEXT }) + .catch((caught: unknown) => caught) + expect(googleCalendarConnector.isListingScopeUnavailableError?.(error)).toBe(true) + }) + + it('still fails the sync under a shared credential rather than dropping the calendar', async () => { + mockCalendars('alpha') + await expect( + googleCalendarConnector.listDocuments('token', { calendarId: 'alpha,beta' }, undefined, {}) + ).rejects.toThrow('Failed to list Google Calendar events: 404') + }) +}) + describe('google-calendar attendee PII opt-out', () => { it('exposes an includeAttendees config field defaulting to on', () => { const field = googleCalendarConnectorMeta.configFields.find((f) => f.id === 'includeAttendees') diff --git a/apps/sim/connectors/google-calendar/google-calendar.ts b/apps/sim/connectors/google-calendar/google-calendar.ts index 921907a82ec..1c61fbccb30 100644 --- a/apps/sim/connectors/google-calendar/google-calendar.ts +++ b/apps/sim/connectors/google-calendar/google-calendar.ts @@ -3,7 +3,13 @@ import { getErrorMessage } from '@sim/utils/errors' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { DEFAULT_MAX_EVENTS, googleCalendarConnectorMeta } from '@/connectors/google-calendar/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { parseMultiValue, parseTagDate } from '@/connectors/utils' +import { + isListingScopeUnavailableError, + isPerMemberListing, + listingRequestError, + parseMultiValue, + parseTagDate, +} from '@/connectors/utils' const logger = createLogger('GoogleCalendarConnector') @@ -301,6 +307,8 @@ function eventToDocument( export const googleCalendarConnector: ConnectorConfig = { ...googleCalendarConnectorMeta, + isListingScopeUnavailableError: isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, @@ -341,9 +349,9 @@ export const googleCalendarConnector: ConnectorConfig = { const calendarId = calendarIds[calendarIndex] const prevFetched = (syncContext?.totalDocsFetched as number) ?? 0 - const rawMaxEvents = sourceConfig.maxEvents - ? Number(sourceConfig.maxEvents) - : DEFAULT_MAX_EVENTS + /** Absent means the default cap; an explicit 0 (a per-member sync) means unlimited. */ + const rawMaxEvents = + sourceConfig.maxEvents === undefined ? DEFAULT_MAX_EVENTS : Number(sourceConfig.maxEvents) const maxEvents = Number.isFinite(rawMaxEvents) ? rawMaxEvents : 0 const isCapped = maxEvents > 0 /** @@ -397,7 +405,33 @@ export const googleCalendarConnector: ConnectorConfig = { calendarId, error: errorText, }) - throw new Error(`Failed to list Google Calendar events: ${response.status}`) + const error = listingRequestError('Failed to list Google Calendar events', response.status) + /** + * One of several calendars a member cannot reach is absent from their + * listing, not the end of it: move on to the next calendar so the rest of + * their access survives. A sole unreachable calendar is the whole scope, + * which the members-mode crawl reads as a complete listing of nothing, and + * a shared credential still fails the sync rather than silently dropping + * the calendar's events. + */ + if ( + isListingScopeUnavailableError(error) && + calendarIds.length > 1 && + isPerMemberListing(syncContext) + ) { + logger.warn('Skipping a Google Calendar the member cannot reach', { + calendarId, + status: response.status, + }) + return calendarIndex + 1 < calendarIds.length + ? { + documents: [], + nextCursor: JSON.stringify({ calendarIndex: calendarIndex + 1 }), + hasMore: true, + } + : { documents: [], hasMore: false } + } + throw error } const data = await response.json() diff --git a/apps/sim/connectors/google-calendar/meta.ts b/apps/sim/connectors/google-calendar/meta.ts index dda94336817..dc9249aa5e3 100644 --- a/apps/sim/connectors/google-calendar/meta.ts +++ b/apps/sim/connectors/google-calendar/meta.ts @@ -16,6 +16,7 @@ export const googleCalendarConnectorMeta: ConnectorMeta = { requiredScopes: ['https://www.googleapis.com/auth/calendar'], }, + permissionScopedListing: { capFieldIds: ['maxEvents'] }, configFields: [ { id: 'calendarSelector', diff --git a/apps/sim/connectors/google-chat/meta.ts b/apps/sim/connectors/google-chat/meta.ts index f20ad56cce0..6da79a86b3c 100644 --- a/apps/sim/connectors/google-chat/meta.ts +++ b/apps/sim/connectors/google-chat/meta.ts @@ -54,6 +54,12 @@ export const googleChatConnectorMeta: ConnectorMeta = { */ rehydrateOnFullSync: true, + /** + * `spaces.list` returns only the spaces the caller is a member of, so one + * member's crawl is exactly what they may read. `maxMessages` bounds each + * space document's window, not which spaces are listed, so it is not a cap. + */ + permissionScopedListing: { capFieldIds: ['maxSpaces'] }, configFields: [ { id: 'spaceTypes', diff --git a/apps/sim/connectors/google-docs/google-docs.ts b/apps/sim/connectors/google-docs/google-docs.ts index 150f807074e..f5a0d2faac7 100644 --- a/apps/sim/connectors/google-docs/google-docs.ts +++ b/apps/sim/connectors/google-docs/google-docs.ts @@ -11,7 +11,9 @@ import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/ import { buildDriveParentsClause, ConnectorFileTooLargeError, + isListingScopeUnavailableError, joinTagArray, + listingRequestError, markSkipped, parseMultiValue, parseOptionalUnlimitedSafeInteger, @@ -493,6 +495,8 @@ function buildQuery(sourceConfig: Record): string { export const googleDocsConnector: ConnectorConfig = { ...googleDocsConnectorMeta, + isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, @@ -553,7 +557,7 @@ export const googleDocsConnector: ConnectorConfig = { status: response.status, error: failure, }) - throw new Error(`Failed to list Google Docs: ${failure}`) + throw listingRequestError(`Failed to list Google Docs: ${failure}`, response.status) } const data = parseDriveFileListResponse(await response.json()) diff --git a/apps/sim/connectors/google-docs/meta.ts b/apps/sim/connectors/google-docs/meta.ts index eaca1bd826b..c3ef01e56dd 100644 --- a/apps/sim/connectors/google-docs/meta.ts +++ b/apps/sim/connectors/google-docs/meta.ts @@ -14,6 +14,7 @@ export const googleDocsConnectorMeta: ConnectorMeta = { requiredScopes: ['https://www.googleapis.com/auth/drive'], }, + permissionScopedListing: { capFieldIds: ['maxDocs'] }, configFields: [ { id: 'folderSelector', diff --git a/apps/sim/connectors/google-drive/google-drive.test.ts b/apps/sim/connectors/google-drive/google-drive.test.ts index 3db406ff869..a1a474bc016 100644 --- a/apps/sim/connectors/google-drive/google-drive.test.ts +++ b/apps/sim/connectors/google-drive/google-drive.test.ts @@ -419,6 +419,21 @@ describe('Google Drive connector limits', () => { expect(syncContext.totalDocsFetched).toBe(1) }) + it('asks only for files modified after an incremental watermark', async () => { + mockFetch.mockResolvedValueOnce(jsonResponse({ files: [] })) + + await googleDriveConnector.listDocuments( + 'token', + {}, + undefined, + {}, + new Date('2026-08-20T12:00:00Z') + ) + + const url = new URL(String(mockFetch.mock.calls[0][0])) + expect(url.searchParams.get('q')).toContain("modifiedTime > '2026-08-20T12:00:00.000Z'") + }) + it('makes an incomplete cross-corpus search non-authoritative', async () => { mockFetch.mockResolvedValueOnce( jsonResponse({ files: [fileMetadata()], incompleteSearch: true }) @@ -501,3 +516,100 @@ describe('Google Drive connector limits', () => { } ) }) + +describe('Google Drive change feed', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mockFetch) + }) + + it("opens the feed at the account's current start token", async () => { + mockFetch.mockResolvedValueOnce(jsonResponse({ startPageToken: '4821' })) + + await expect(googleDriveConnector.getChangeCursor?.('token', {})).resolves.toBe('4821') + expect(String(mockFetch.mock.calls[0][0])).toContain('/changes/startPageToken') + }) + + it('reports lost access and trashed files as removals and in-scope files as upserts', async () => { + mockFetch.mockResolvedValueOnce( + jsonResponse({ + changes: [ + { changeType: 'file', fileId: 'gone', removed: true }, + { + changeType: 'file', + fileId: 'binned', + file: fileMetadata({ id: 'binned', trashed: true }), + }, + { + changeType: 'file', + fileId: 'kept', + file: fileMetadata({ id: 'kept', parents: ['f-1'] }), + }, + { + changeType: 'file', + fileId: 'moved-out', + file: fileMetadata({ id: 'moved-out', parents: ['elsewhere'] }), + }, + { + changeType: 'file', + fileId: 'video', + file: fileMetadata({ id: 'video', mimeType: 'video/mp4', parents: ['f-1'] }), + }, + { changeType: 'drive', driveId: 'd-1' }, + ], + newStartPageToken: '5000', + }) + ) + + const result = await googleDriveConnector.listChanges!('token', { folderId: 'f-1' }, '4821') + + expect(result.changes).toEqual([ + { kind: 'removed', externalId: 'gone' }, + { kind: 'removed', externalId: 'binned' }, + { + kind: 'upsert', + externalId: 'kept', + document: expect.objectContaining({ externalId: 'kept' }), + }, + { kind: 'removed', externalId: 'moved-out' }, + { kind: 'removed', externalId: 'video' }, + ]) + expect(result.nextCursor).toBe('5000') + expect(result.hasMore).toBe(false) + const url = new URL(String(mockFetch.mock.calls[0][0])) + expect(url.searchParams.get('pageToken')).toBe('4821') + expect(url.searchParams.get('includeRemoved')).toBe('true') + }) + + it('continues on the next page token while the feed has more', async () => { + mockFetch.mockResolvedValueOnce( + jsonResponse({ changes: [], nextPageToken: '4900', newStartPageToken: '5000' }) + ) + + const result = await googleDriveConnector.listChanges!('token', {}, '4821') + + expect(result).toEqual({ changes: [], nextCursor: '4900', hasMore: true }) + }) + + it('rejects a feed page without a cursor to continue from', async () => { + mockFetch.mockResolvedValueOnce(jsonResponse({ changes: [] })) + + await expect(googleDriveConnector.listChanges!('token', {}, '4821')).rejects.toThrow( + 'malformed change-list metadata' + ) + }) + + it.each([ + [400, [], true], + [400, ['invalid'], true], + [404, ['notFound'], true], + [410, [], true], + [403, ['insufficientFilePermissions'], false], + [500, ['backendError'], false], + ])('classifies HTTP %s %j as cursor-invalid=%s', (status, reasons, expected) => { + expect( + googleDriveConnector.isChangeCursorInvalidError!(new GoogleDriveApiError(status, reasons)) + ).toBe(expected) + expect(googleDriveConnector.isChangeCursorInvalidError!(new Error('other'))).toBe(false) + }) +}) diff --git a/apps/sim/connectors/google-drive/google-drive.ts b/apps/sim/connectors/google-drive/google-drive.ts index fee472e9f7f..b10af7fa341 100644 --- a/apps/sim/connectors/google-drive/google-drive.ts +++ b/apps/sim/connectors/google-drive/google-drive.ts @@ -14,7 +14,13 @@ import { readGoogleDriveApiError, } from '@/connectors/google-drive/google-drive-errors' import { googleDriveConnectorMeta } from '@/connectors/google-drive/meta' -import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' +import type { + ConnectorConfig, + ExternalChange, + ExternalChangeList, + ExternalDocument, + ExternalDocumentList, +} from '@/connectors/types' import { buildDriveParentsClause, CONNECTOR_MAX_FILE_BYTES, @@ -203,6 +209,20 @@ interface DriveFile { size?: string starred?: boolean trashed?: boolean + parents?: string[] +} + +interface DriveChange { + changeType?: string + removed?: boolean + fileId?: string + file?: DriveFile +} + +interface DriveChangeListResponse { + changes: DriveChange[] + nextPageToken?: string + newStartPageToken?: string } interface DriveFileListResponse { @@ -273,12 +293,120 @@ function parseDriveFileMetadata(value: unknown, expectedId: string): DriveFile { return value } -function buildQuery(sourceConfig: Record): string { +function parseDriveChangeListResponse(value: unknown): DriveChangeListResponse { + if (!isPlainRecord(value)) { + throw new Error('Google Drive API returned malformed change-list metadata') + } + const rawChanges = value.changes + if (rawChanges !== undefined && !Array.isArray(rawChanges)) { + throw new Error('Google Drive API returned malformed change-list metadata') + } + const changes: DriveChange[] = [] + for (const raw of rawChanges ?? []) { + if (!isPlainRecord(raw) || typeof raw.fileId !== 'string' || raw.fileId.length === 0) { + /** Shared-drive membership changes carry no fileId and are not files. */ + if (isPlainRecord(raw) && raw.changeType === 'drive') continue + throw new Error('Google Drive API returned malformed change-list metadata') + } + if (raw.file !== undefined && !isDriveFileListItem(raw.file)) { + throw new Error('Google Drive API returned malformed change-list metadata') + } + changes.push({ + changeType: typeof raw.changeType === 'string' ? raw.changeType : undefined, + removed: raw.removed === true, + fileId: raw.fileId, + file: raw.file, + }) + } + for (const key of ['nextPageToken', 'newStartPageToken'] as const) { + const token = value[key] + if (token !== undefined && (typeof token !== 'string' || token.length === 0)) { + throw new Error('Google Drive API returned malformed change-list metadata') + } + } + return { + changes, + nextPageToken: typeof value.nextPageToken === 'string' ? value.nextPageToken : undefined, + newStartPageToken: + typeof value.newStartPageToken === 'string' ? value.newStartPageToken : undefined, + } +} + +/** The MIME types the `fileType` setting admits, mirroring {@link buildQuery}. */ +function matchesFileType(fileType: string, mimeType: string): boolean { + switch (fileType) { + case 'documents': + return mimeType === 'application/vnd.google-apps.document' + case 'spreadsheets': + return mimeType === 'application/vnd.google-apps.spreadsheet' + case 'presentations': + return mimeType === 'application/vnd.google-apps.presentation' + case 'text': + return SUPPORTED_TEXT_MIME_TYPES.includes(mimeType) + default: + return isGoogleWorkspaceFile(mimeType) || isSupportedTextFile(mimeType) + } +} + +/** + * Whether a file reported by the change feed belongs to the configured + * source. A listing applies these as a query; the feed reports every change + * the account can see, so they are applied here instead. A file that left the + * scope reads as removed, exactly as a listing would no longer return it. + */ +function isFileInScope(file: DriveFile, sourceConfig: Record): boolean { + if (file.trashed) return false + if (!matchesFileType((sourceConfig.fileType as string) || 'all', file.mimeType)) return false + const folderIds = parseMultiValue(sourceConfig.folderId) + if (folderIds.length === 0) return true + return (file.parents ?? []).some((parent) => folderIds.includes(parent)) +} + +function driveChangeToExternal( + change: DriveChange, + sourceConfig: Record +): ExternalChange | null { + if (change.changeType !== undefined && change.changeType !== 'file') return null + const externalId = change.fileId + if (!externalId) return null + const file = change.file + if (change.removed || !file || !isFileInScope(file, sourceConfig)) { + return { kind: 'removed', externalId } + } + return { + kind: 'upsert', + externalId, + document: stubOrSkipBySize( + fileToStub(file), + Number(file.size) || undefined, + CONNECTOR_MAX_FILE_BYTES + ), + } +} + +/** + * Drive rejects an expired or foreign page token as a bad request rather than + * with a dedicated status; a 404 or 410 is the same signal on other endpoints. + * Reopening the feed from a full listing is the safe answer to all of them. + */ +function isDriveChangeCursorInvalidError(error: unknown): boolean { + if (!(error instanceof GoogleDriveApiError)) return false + if (error.status === 404 || error.status === 410) return true + return ( + error.status === 400 && + (error.reasons.length === 0 || + error.reasons.some((reason) => reason === 'invalid' || reason === 'badRequest')) + ) +} + +function buildQuery(sourceConfig: Record, lastSyncAt?: Date): string { const parts: string[] = ['trashed = false'] const parentsClause = buildDriveParentsClause(parseMultiValue(sourceConfig.folderId)) if (parentsClause) parts.push(parentsClause) + if (lastSyncAt) parts.push(`modifiedTime > '${lastSyncAt.toISOString()}'`) + const fileType = (sourceConfig.fileType as string) || 'all' switch (fileType) { case 'documents': @@ -339,9 +467,10 @@ export const googleDriveConnector: ConnectorConfig = { accessToken: string, sourceConfig: Record, cursor?: string, - syncContext?: Record + syncContext?: Record, + lastSyncAt?: Date ): Promise => { - const query = buildQuery(sourceConfig) + const query = buildQuery(sourceConfig, lastSyncAt) const pageSize = 100 const maxFiles = parseMaxFiles(sourceConfig.maxFiles) @@ -359,7 +488,7 @@ export const googleDriveConnector: ConnectorConfig = { pageSize: String(effectivePageSize), orderBy: 'modifiedTime desc', fields: - 'kind,nextPageToken,incompleteSearch,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,owners,size,starred)', + 'kind,nextPageToken,incompleteSearch,files(id,name,mimeType,modifiedTime,createdTime,webViewLink,owners,size,starred,parents)', supportsAllDrives: 'true', includeItemsFromAllDrives: 'true', }) @@ -612,4 +741,78 @@ export const googleDriveConnector: ConnectorConfig = { return result }, + + /** + * Drive answers `notFound` for a `parents` query on a folder the caller + * cannot open, so a member who was never given the folder lists nothing. + */ + isListingScopeUnavailableError: (error) => + error instanceof GoogleDriveApiError && error.kind === 'not_found', + + getChangeCursor: async (accessToken: string): Promise => { + const url = 'https://www.googleapis.com/drive/v3/changes/startPageToken?supportsAllDrives=true' + const response = await fetchGoogleDriveWithRetry(url, { + method: 'GET', + headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, + }) + const data: unknown = await response.json() + if ( + !isPlainRecord(data) || + typeof data.startPageToken !== 'string' || + data.startPageToken.length === 0 + ) { + throw new Error('Google Drive API returned malformed change-cursor metadata') + } + return data.startPageToken + }, + + /** + * Reads `changes.list` for the account behind the token. Drive reports a + * file the account lost access to with `removed: true`, and a file newly + * shared with it as an ordinary change, so one feed carries both content + * and permission changes for that account. + */ + listChanges: async ( + accessToken: string, + sourceConfig: Record, + cursor: string + ): Promise => { + const queryParams = new URLSearchParams({ + pageToken: cursor, + pageSize: '100', + includeRemoved: 'true', + supportsAllDrives: 'true', + includeItemsFromAllDrives: 'true', + restrictToMyDrive: 'false', + spaces: 'drive', + fields: + 'nextPageToken,newStartPageToken,changes(changeType,removed,fileId,file(id,name,mimeType,modifiedTime,createdTime,webViewLink,owners,size,starred,trashed,parents))', + }) + const url = `https://www.googleapis.com/drive/v3/changes?${queryParams.toString()}` + + let response: Response + try { + response = await fetchGoogleDriveWithRetry(url, { + method: 'GET', + headers: { Authorization: `Bearer ${accessToken}`, Accept: 'application/json' }, + }) + } catch (error) { + logger.error('Failed to list Google Drive changes', googleDriveErrorLogFields(error)) + throw error + } + + const data = parseDriveChangeListResponse(await response.json()) + const changes: ExternalChange[] = [] + for (const change of data.changes) { + const mapped = driveChangeToExternal(change, sourceConfig) + if (mapped) changes.push(mapped) + } + const nextCursor = data.nextPageToken ?? data.newStartPageToken + if (!nextCursor) { + throw new Error('Google Drive API returned malformed change-list metadata') + } + return { changes, nextCursor, hasMore: Boolean(data.nextPageToken) } + }, + + isChangeCursorInvalidError: isDriveChangeCursorInvalidError, } diff --git a/apps/sim/connectors/google-drive/meta.ts b/apps/sim/connectors/google-drive/meta.ts index eec6fd8fa2e..4b0ccd2cf25 100644 --- a/apps/sim/connectors/google-drive/meta.ts +++ b/apps/sim/connectors/google-drive/meta.ts @@ -14,6 +14,9 @@ export const googleDriveConnectorMeta: ConnectorMeta = { requiredScopes: ['https://www.googleapis.com/auth/drive'], }, + /** `files.list` under a member's token returns only what that member can open. */ + permissionScopedListing: { capFieldIds: ['maxFiles'] }, + configFields: [ { id: 'folderSelector', diff --git a/apps/sim/connectors/google-forms/google-forms.ts b/apps/sim/connectors/google-forms/google-forms.ts index 443b394443a..957e039cb93 100644 --- a/apps/sim/connectors/google-forms/google-forms.ts +++ b/apps/sim/connectors/google-forms/google-forms.ts @@ -5,7 +5,9 @@ import { googleFormsConnectorMeta, MAX_RESPONSES_PER_FORM } from '@/connectors/g import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { buildDriveParentsClause, + isListingScopeUnavailableError, joinTagArray, + listingRequestError, parseMultiValue, parseTagDate, } from '@/connectors/utils' @@ -434,6 +436,8 @@ function buildDriveQuery(folderIds: string[]): string { export const googleFormsConnector: ConnectorConfig = { ...googleFormsConnectorMeta, + isListingScopeUnavailableError: isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, @@ -480,7 +484,7 @@ export const googleFormsConnector: ConnectorConfig = { if (!response.ok) { const errorText = await response.text() logger.error('Failed to list Google Forms', { status: response.status, error: errorText }) - throw new Error(`Failed to list Google Forms: ${response.status}`) + throw listingRequestError('Failed to list Google Forms', response.status) } const data = await response.json() diff --git a/apps/sim/connectors/google-forms/meta.ts b/apps/sim/connectors/google-forms/meta.ts index 3ea4b31fbdb..73f87cef9bf 100644 --- a/apps/sim/connectors/google-forms/meta.ts +++ b/apps/sim/connectors/google-forms/meta.ts @@ -24,6 +24,7 @@ export const googleFormsConnectorMeta: ConnectorMeta = { ], }, + permissionScopedListing: { capFieldIds: ['maxForms'] }, configFields: [ { id: 'folderSelector', diff --git a/apps/sim/connectors/google-meet/meta.ts b/apps/sim/connectors/google-meet/meta.ts index 88112e590d4..0b5f6afa5fe 100644 --- a/apps/sim/connectors/google-meet/meta.ts +++ b/apps/sim/connectors/google-meet/meta.ts @@ -14,6 +14,13 @@ export const googleMeetConnectorMeta: ConnectorMeta = { requiredScopes: ['https://www.googleapis.com/auth/meetings.space.readonly'], }, + /** + * `conferenceRecords.list` returns only the conferences the caller + * organized, so a member's crawl never reaches a meeting they cannot read. + * It also omits meetings they merely attended, the same shape as Zoom's + * own-recordings listing. + */ + permissionScopedListing: { capFieldIds: ['maxMeetings'] }, configFields: [ { id: 'maxMeetings', diff --git a/apps/sim/connectors/google-sheets/google-sheets.test.ts b/apps/sim/connectors/google-sheets/google-sheets.test.ts index 56c0d9b540c..8279753c025 100644 --- a/apps/sim/connectors/google-sheets/google-sheets.test.ts +++ b/apps/sim/connectors/google-sheets/google-sheets.test.ts @@ -89,6 +89,7 @@ interface FetchStubResponses { drive: { status: number; body: unknown } values?: unknown spreadsheet?: unknown + spreadsheetStatus?: number } /** @@ -110,7 +111,7 @@ function stubFetch(responses: FetchStubResponses) { } if (url.startsWith('https://sheets.googleapis.com/v4/spreadsheets/')) { return new Response(JSON.stringify(responses.spreadsheet ?? SPREADSHEET_METADATA), { - status: 200, + status: responses.spreadsheetStatus ?? 200, }) } throw new Error(`Unexpected fetch to ${url}`) @@ -178,6 +179,38 @@ describe('googleSheetsConnector trashed handling', () => { expect(result.documents).toHaveLength(2) }) + + it.each([403, 404])( + 'reports a spreadsheet the token cannot reach (%i) as an unavailable listing scope', + async (status) => { + stubFetch({ + drive: { status: 200, body: {} }, + spreadsheet: { error: 'denied' }, + spreadsheetStatus: status, + }) + + const error = await googleSheetsConnector + .listDocuments(ACCESS_TOKEN, SOURCE_CONFIG) + .catch((caught: unknown) => caught) + + expect(googleSheetsConnector.isListingScopeUnavailableError?.(error)).toBe(true) + } + ) + + it('keeps any other metadata failure retryable', async () => { + stubFetch({ + drive: { status: 200, body: {} }, + spreadsheet: { error: 'backend' }, + spreadsheetStatus: 500, + }) + + const error = await googleSheetsConnector + .listDocuments(ACCESS_TOKEN, SOURCE_CONFIG) + .catch((caught: unknown) => caught) + + expect(error).toBeInstanceOf(Error) + expect(googleSheetsConnector.isListingScopeUnavailableError?.(error)).toBe(false) + }) }) describe('getDocument', () => { diff --git a/apps/sim/connectors/google-sheets/google-sheets.ts b/apps/sim/connectors/google-sheets/google-sheets.ts index 79cd8c6ad77..aba40c3d6f8 100644 --- a/apps/sim/connectors/google-sheets/google-sheets.ts +++ b/apps/sim/connectors/google-sheets/google-sheets.ts @@ -6,6 +6,8 @@ import { googleSheetsConnectorMeta } from '@/connectors/google-sheets/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { CONNECTOR_MAX_FILE_BYTES, + ConnectorListingScopeUnavailableError, + isListingScopeUnavailableError, markSkipped, parseTagDate, readBodyWithLimit, @@ -172,7 +174,15 @@ async function fetchSpreadsheetMetadata( }) if (!response.ok) { - throw new Error(`Failed to fetch spreadsheet metadata: ${response.status}`) + const message = `Failed to fetch spreadsheet metadata: ${response.status}` + /** + * The Sheets API answers a spreadsheet that is not shared with the caller + * with 403, and one they cannot see at all with 404: either way the + * configured spreadsheet is out of this caller's reach. + */ + throw response.status === 403 || response.status === 404 + ? new ConnectorListingScopeUnavailableError(message, response.status) + : new Error(message) } return (await response.json()) as SpreadsheetMetadata @@ -402,6 +412,8 @@ async function sheetToDocument( export const googleSheetsConnector: ConnectorConfig = { ...googleSheetsConnectorMeta, + isListingScopeUnavailableError: isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, diff --git a/apps/sim/connectors/google-sheets/meta.ts b/apps/sim/connectors/google-sheets/meta.ts index 7a20991d719..2597c4f2e39 100644 --- a/apps/sim/connectors/google-sheets/meta.ts +++ b/apps/sim/connectors/google-sheets/meta.ts @@ -14,6 +14,11 @@ export const googleSheetsConnectorMeta: ConnectorMeta = { requiredScopes: ['https://www.googleapis.com/auth/drive'], }, + /** + * The listing is one configured spreadsheet's tabs: a member who can read + * the file lists every tab, one who cannot lists nothing. Nothing caps it. + */ + permissionScopedListing: { capFieldIds: [] }, configFields: [ { id: 'spreadsheetSelector', diff --git a/apps/sim/connectors/google-slides/google-slides.ts b/apps/sim/connectors/google-slides/google-slides.ts index cd957564e99..1257cf60aa5 100644 --- a/apps/sim/connectors/google-slides/google-slides.ts +++ b/apps/sim/connectors/google-slides/google-slides.ts @@ -7,7 +7,9 @@ import { buildDriveParentsClause, CONNECTOR_MAX_FILE_BYTES, ConnectorFileTooLargeError, + isListingScopeUnavailableError, joinTagArray, + listingRequestError, markSkipped, parseMultiValue, parseTagDate, @@ -286,6 +288,8 @@ function buildQuery(sourceConfig: Record, lastSyncAt?: Date): s export const googleSlidesConnector: ConnectorConfig = { ...googleSlidesConnectorMeta, + isListingScopeUnavailableError: isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, @@ -339,7 +343,7 @@ export const googleSlidesConnector: ConnectorConfig = { status: response.status, error: errorText, }) - throw new Error(`Failed to list Google Slides presentations: ${response.status}`) + throw listingRequestError('Failed to list Google Slides presentations', response.status) } const data = await response.json() diff --git a/apps/sim/connectors/google-slides/meta.ts b/apps/sim/connectors/google-slides/meta.ts index a4cc9a17aef..72242deef12 100644 --- a/apps/sim/connectors/google-slides/meta.ts +++ b/apps/sim/connectors/google-slides/meta.ts @@ -19,6 +19,7 @@ export const googleSlidesConnectorMeta: ConnectorMeta = { requiredScopes: ['https://www.googleapis.com/auth/drive'], }, + permissionScopedListing: { capFieldIds: ['maxDocs'] }, configFields: [ { id: 'folderSelector', diff --git a/apps/sim/connectors/jira/jira.ts b/apps/sim/connectors/jira/jira.ts index a957b3f7468..44e16893f5f 100644 --- a/apps/sim/connectors/jira/jira.ts +++ b/apps/sim/connectors/jira/jira.ts @@ -1,10 +1,20 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' -import { normalizeAtlassianSiteUrl } from '@/lib/atlassian/discovery' +import { + AtlassianSiteNotAccessibleError, + AtlassianSiteNotMatchedError, + normalizeAtlassianSiteUrl, +} from '@/lib/atlassian/discovery' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { jiraConnectorMeta } from '@/connectors/jira/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { joinTagArray, parseMultiValue, parseTagDate } from '@/connectors/utils' +import { + isListingScopeUnavailableError, + joinTagArray, + listingRequestError, + parseMultiValue, + parseTagDate, +} from '@/connectors/utils' import { extractAdfText, getJiraCloudId } from '@/tools/jira/utils' const logger = createLogger('JiraConnector') @@ -131,6 +141,15 @@ function issueToFullDocument(issue: Record, siteUrl: string): E export const jiraConnector: ConnectorConfig = { ...jiraConnectorMeta, + /** + * A member whose token reaches no Atlassian site, or only sites other than + * the configured one, lists nothing: a complete listing of nothing, not an error. + */ + isListingScopeUnavailableError: (error) => + isListingScopeUnavailableError(error) || + error instanceof AtlassianSiteNotAccessibleError || + error instanceof AtlassianSiteNotMatchedError, + listDocuments: async ( accessToken: string, sourceConfig: Record, @@ -219,7 +238,12 @@ export const jiraConnector: ConnectorConfig = { status: response.status, error: errorText, }) - throw new Error(`Failed to search Jira issues: ${response.status}`) + throw listingRequestError( + 'Failed to search Jira issues', + response.status, + response.status === 404 || + (response.status === 400 && /does not exist for the field 'project'/i.test(errorText)) + ) } const data = await response.json() diff --git a/apps/sim/connectors/jira/meta.ts b/apps/sim/connectors/jira/meta.ts index 40c1749618a..d38ee66c672 100644 --- a/apps/sim/connectors/jira/meta.ts +++ b/apps/sim/connectors/jira/meta.ts @@ -10,6 +10,7 @@ export const jiraConnectorMeta: ConnectorMeta = { auth: { mode: 'oauth', provider: 'jira', requiredScopes: ['read:jira-work', 'offline_access'] }, + permissionScopedListing: { capFieldIds: ['maxIssues'] }, configFields: [ { id: 'domain', diff --git a/apps/sim/connectors/jsm/jsm.test.ts b/apps/sim/connectors/jsm/jsm.test.ts new file mode 100644 index 00000000000..6aab2f5e135 --- /dev/null +++ b/apps/sim/connectors/jsm/jsm.test.ts @@ -0,0 +1,75 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() })) + +vi.mock('@/lib/knowledge/documents/utils', () => ({ + fetchWithRetry: mockFetchWithRetry, + VALIDATE_RETRY_OPTIONS: {}, +})) +vi.mock('@/components/icons', () => ({ JiraServiceManagementIcon: () => null })) +vi.mock('@/tools/jira/utils', () => ({ + getJiraCloudId: vi.fn(), + extractAdfText: () => '', +})) + +import { AtlassianSiteNotMatchedError } from '@/lib/atlassian/discovery' +import { jsmConnector } from '@/connectors/jsm/jsm' + +const SOURCE_CONFIG = { domain: 'example.atlassian.net', serviceDeskId: '10' } + +function mockStatus(status: number) { + mockFetchWithRetry.mockResolvedValue({ + ok: status >= 200 && status < 300, + status, + json: async () => ({}), + text: async () => '', + } as unknown as Response) +} + +async function listingError(): Promise { + return jsmConnector + .listDocuments('token', SOURCE_CONFIG, undefined, { cloudId: 'cloud-1' }) + .catch((caught: unknown) => caught) +} + +beforeEach(() => { + vi.clearAllMocks() +}) + +describe('jsm listing scope classification', () => { + it('treats a 403 on the request listing as a service desk the caller may not view', async () => { + mockStatus(403) + expect(jsmConnector.isListingScopeUnavailableError?.(await listingError())).toBe(true) + }) + + it('treats a 404 on the request listing as a service desk that does not exist for the caller', async () => { + mockStatus(404) + expect(jsmConnector.isListingScopeUnavailableError?.(await listingError())).toBe(true) + }) + + it('leaves other failures for the sync engines to retry', async () => { + mockStatus(500) + const error = await listingError() + expect(error).toBeInstanceOf(Error) + expect(jsmConnector.isListingScopeUnavailableError?.(error)).toBe(false) + }) + + it('treats a 403 while resolving a project key to a service desk id the same way', async () => { + mockStatus(403) + const error = await jsmConnector + .listDocuments('token', { ...SOURCE_CONFIG, serviceDeskId: 'ITH' }, undefined, { + cloudId: 'cloud-1', + }) + .catch((caught: unknown) => caught) + expect(jsmConnector.isListingScopeUnavailableError?.(error)).toBe(true) + }) + + it('treats a token that reaches only other Atlassian sites as not on the site', () => { + expect( + jsmConnector.isListingScopeUnavailableError?.(new AtlassianSiteNotMatchedError('elsewhere')) + ).toBe(true) + }) +}) diff --git a/apps/sim/connectors/jsm/jsm.ts b/apps/sim/connectors/jsm/jsm.ts index ee42607ed9a..a1a05066dca 100644 --- a/apps/sim/connectors/jsm/jsm.ts +++ b/apps/sim/connectors/jsm/jsm.ts @@ -1,9 +1,18 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' +import { + AtlassianSiteNotAccessibleError, + AtlassianSiteNotMatchedError, +} from '@/lib/atlassian/discovery' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { jsmConnectorMeta } from '@/connectors/jsm/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { htmlToPlainText, parseTagDate } from '@/connectors/utils' +import { + htmlToPlainText, + isListingScopeUnavailableError, + listingRequestError, + parseTagDate, +} from '@/connectors/utils' import { extractAdfText, getJiraCloudId } from '@/tools/jira/utils' import { getJsmApiBaseUrl, getJsmHeaders } from '@/tools/jsm/utils' @@ -305,6 +314,15 @@ async function resolveCloudId( return cloudId } +/** + * JSM answers a service desk the caller may not view with 403 and one that + * does not exist for them with 404; either is a complete listing of nothing + * for that caller. + */ +function isJsmScopeUnavailableStatus(status: number): boolean { + return status === 403 || status === 404 +} + /** * Resolves a configured service desk identifier to the numeric service desk id. * @@ -331,7 +349,11 @@ async function resolveServiceDeskId( }) if (!response.ok) { - throw new Error(`Failed to resolve service desk "${trimmed}": ${response.status}`) + throw listingRequestError( + `Failed to resolve service desk "${trimmed}"`, + response.status, + isJsmScopeUnavailableStatus(response.status) + ) } const data = (await response.json()) as { id?: string } @@ -421,6 +443,16 @@ async function fetchComments( export const jsmConnector: ConnectorConfig = { ...jsmConnectorMeta, + /** + * A member whose token reaches no Atlassian site, or only sites other than + * the configured one, or who may not view the configured service desk, lists + * nothing: a complete listing of nothing, not an error. + */ + isListingScopeUnavailableError: (error) => + isListingScopeUnavailableError(error) || + error instanceof AtlassianSiteNotAccessibleError || + error instanceof AtlassianSiteNotMatchedError, + listDocuments: async ( accessToken: string, sourceConfig: Record, @@ -498,7 +530,11 @@ export const jsmConnector: ConnectorConfig = { if (!response.ok) { const errorText = await response.text() logger.error('Failed to list JSM requests', { status: response.status, error: errorText }) - throw new Error(`Failed to list JSM requests: ${response.status}`) + throw listingRequestError( + 'Failed to list JSM requests', + response.status, + isJsmScopeUnavailableStatus(response.status) + ) } const data = (await response.json()) as JsmPage diff --git a/apps/sim/connectors/jsm/meta.ts b/apps/sim/connectors/jsm/meta.ts index 8b4f79003f8..22c9cd4b748 100644 --- a/apps/sim/connectors/jsm/meta.ts +++ b/apps/sim/connectors/jsm/meta.ts @@ -36,6 +36,7 @@ export const jsmConnectorMeta: ConnectorMeta = { ], }, + permissionScopedListing: { capFieldIds: ['maxRequests'] }, configFields: [ { id: 'domain', diff --git a/apps/sim/connectors/linear/linear.ts b/apps/sim/connectors/linear/linear.ts index 93aab253e64..47536312153 100644 --- a/apps/sim/connectors/linear/linear.ts +++ b/apps/sim/connectors/linear/linear.ts @@ -6,7 +6,13 @@ import type { RetryOptions } from '@/lib/knowledge/documents/utils' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { linearConnectorMeta } from '@/connectors/linear/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { joinTagArray, parseMultiValue, parseTagDate } from '@/connectors/utils' +import { + ConnectorListingScopeUnavailableError, + isListingScopeUnavailableError, + joinTagArray, + parseMultiValue, + parseTagDate, +} from '@/connectors/utils' import { linearAuthorizationHeader } from '@/tools/linear/utils' const logger = createLogger('LinearConnector') @@ -55,6 +61,15 @@ const MAX_RATE_LIMIT_WAIT_MS = 30_000 /** * Detects Linear's `RATELIMITED` extension code anywhere in a GraphQL error array. */ +function isEntityNotFoundError(entry: unknown): boolean { + if (!entry || typeof entry !== 'object') return false + const error = entry as { message?: unknown; extensions?: { code?: unknown } } + return ( + error.extensions?.code === 'ENTITY_NOT_FOUND' || + (typeof error.message === 'string' && /entity not found/i.test(error.message)) + ) +} + function isRateLimitedErrors(errors: unknown[] | undefined): boolean { if (!Array.isArray(errors)) return false return errors.some((entry) => { @@ -129,7 +144,11 @@ async function linearGraphQL( */ if (Array.isArray(json?.errors) && json.errors.length > 0) { logger.error('Linear GraphQL errors', { errors: json.errors }) - throw new Error(`Linear GraphQL error: ${JSON.stringify(json.errors)}`) + const described = `Linear GraphQL error: ${JSON.stringify(json.errors)}` + /** A team or project the caller cannot see is reported as an entity that does not exist. */ + throw json.errors.some(isEntityNotFoundError) + ? new ConnectorListingScopeUnavailableError(described, response.status) + : new Error(described) } if (!json?.data || typeof json.data !== 'object') { @@ -297,6 +316,8 @@ function issueToDocument(issue: Record): ExternalDocument { export const linearConnector: ConnectorConfig = { ...linearConnectorMeta, + isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, diff --git a/apps/sim/connectors/linear/meta.ts b/apps/sim/connectors/linear/meta.ts index 33f9e4604ff..070346a5e13 100644 --- a/apps/sim/connectors/linear/meta.ts +++ b/apps/sim/connectors/linear/meta.ts @@ -18,6 +18,7 @@ export const linearConnectorMeta: ConnectorMeta = { */ supportsIncrementalSync: true, + permissionScopedListing: { capFieldIds: ['maxIssues'] }, configFields: [ { id: 'teamSelector', diff --git a/apps/sim/connectors/microsoft-excel/meta.ts b/apps/sim/connectors/microsoft-excel/meta.ts index 05ac3517d45..463993e4070 100644 --- a/apps/sim/connectors/microsoft-excel/meta.ts +++ b/apps/sim/connectors/microsoft-excel/meta.ts @@ -14,6 +14,12 @@ export const microsoftExcelConnectorMeta: ConnectorMeta = { requiredScopes: ['Files.ReadWrite'], }, + /** + * No config field caps the listing: every worksheet of the one workbook is + * listed. The `MAX_WORKSHEETS` memory bound flags `listingCapped` when it + * bites, which the members-mode crawl reads as an incomplete listing. + */ + permissionScopedListing: { capFieldIds: [] }, configFields: [ { id: 'driveId', diff --git a/apps/sim/connectors/microsoft-excel/microsoft-excel.ts b/apps/sim/connectors/microsoft-excel/microsoft-excel.ts index aa28a9e2696..e9085e137a2 100644 --- a/apps/sim/connectors/microsoft-excel/microsoft-excel.ts +++ b/apps/sim/connectors/microsoft-excel/microsoft-excel.ts @@ -4,7 +4,13 @@ import type { RetryOptions } from '@/lib/knowledge/documents/utils' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { microsoftExcelConnectorMeta } from '@/connectors/microsoft-excel/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { markSkipped, parseTagDate, readBodyWithLimit } from '@/connectors/utils' +import { + ConnectorListingScopeUnavailableError, + isListingScopeUnavailableError, + markSkipped, + parseTagDate, + readBodyWithLimit, +} from '@/connectors/utils' import type { ExcelCellValue } from '@/tools/microsoft_excel/types' import { escapeODataString, @@ -221,11 +227,18 @@ function worksheetUrl(basePath: string, sheetName: string): string { return `${basePath}/workbook/worksheets('${encodeURIComponent(escapeODataString(sheetName))}')` } -/** Throws a Graph-formatted error for a failed response. */ +/** + * Throws a Graph-formatted error for a failed response. A workbook Graph will + * not open for the caller (403 `accessDenied`, 404 `itemNotFound`) is a scope + * they cannot reach rather than a fault to retry. + */ async function graphError(response: Response, context: string): Promise { const body = await response.text().catch(() => '') const detail = parseGraphErrorMessage(response.status, response.statusText, body) - throw new Error(`${context}: ${detail}`) + const message = `${context}: ${detail}` + throw response.status === 403 || response.status === 404 + ? new ConnectorListingScopeUnavailableError(message, response.status) + : new Error(message) } /** Fetches the workbook drive item (name, webUrl, lastModifiedDateTime). */ @@ -490,6 +503,8 @@ function resolveBasePath(sourceConfig: Record): { export const microsoftExcelConnector: ConnectorConfig = { ...microsoftExcelConnectorMeta, + isListingScopeUnavailableError: isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, diff --git a/apps/sim/connectors/microsoft-teams/meta.ts b/apps/sim/connectors/microsoft-teams/meta.ts index 5cb79cea453..e0b943ddd55 100644 --- a/apps/sim/connectors/microsoft-teams/meta.ts +++ b/apps/sim/connectors/microsoft-teams/meta.ts @@ -21,6 +21,11 @@ export const microsoftTeamsConnectorMeta: ConnectorMeta = { requiredScopes: ['ChannelMessage.Read.All', 'Channel.ReadBasic.All', 'Team.ReadBasic.All'], }, + /** + * `maxMessages` bounds how much history each channel document carries, not + * which channels are listed, so a member's listing is complete under any value. + */ + permissionScopedListing: { capFieldIds: [] }, configFields: [ { id: 'teamSelector', diff --git a/apps/sim/connectors/microsoft-teams/microsoft-teams.test.ts b/apps/sim/connectors/microsoft-teams/microsoft-teams.test.ts new file mode 100644 index 00000000000..306289f09c5 --- /dev/null +++ b/apps/sim/connectors/microsoft-teams/microsoft-teams.test.ts @@ -0,0 +1,156 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() })) + +vi.mock('@/lib/knowledge/documents/utils', () => ({ + fetchWithRetry: mockFetchWithRetry, + VALIDATE_RETRY_OPTIONS: {}, +})) +vi.mock('@/components/icons', () => ({ MicrosoftTeamsIcon: () => null })) + +import { microsoftTeamsConnector } from '@/connectors/microsoft-teams/microsoft-teams' +import { PER_MEMBER_LISTING_CONTEXT } from '@/connectors/utils' + +const GRAPH = 'https://graph.microsoft.com/v1.0' +const TEAM_ID = 'team-1' +const CHANNELS_URL = `${GRAPH}/teams/${TEAM_ID}/channels?$select=id,displayName,description` + +interface GraphRoute { + status?: number + body?: unknown +} + +/** Installs a URL-keyed fake Graph; unrouted URLs reply 404. */ +function mockGraph(routes: Record) { + mockFetchWithRetry.mockImplementation(async (url: string) => { + const route = routes[url] ?? { status: 404 } + const status = route.status ?? 200 + return { + ok: status >= 200 && status < 300, + status, + json: async () => route.body, + text: async () => JSON.stringify(route.body ?? {}), + } as unknown as Response + }) +} + +async function listingError(): Promise { + return microsoftTeamsConnector + .listDocuments('token', { teamId: TEAM_ID, channel: 'General' }) + .catch((error: unknown) => error) +} + +describe('microsoft teams listing scope', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it.each([403, 404])( + 'reads a %s on the configured team as a scope the caller cannot reach', + async (status) => { + mockGraph({ [CHANNELS_URL]: { status, body: {} } }) + + const error = await listingError() + + expect(error).toBeInstanceOf(Error) + expect(microsoftTeamsConnector.isListingScopeUnavailableError!(error)).toBe(true) + } + ) + + it('reads a channel the caller cannot see as a scope they cannot reach', async () => { + mockGraph({ + [CHANNELS_URL]: { body: { value: [{ id: 'c1', displayName: 'Announcements' }] } }, + }) + + const error = await listingError() + + expect(error).toBeInstanceOf(Error) + expect(String(error)).toMatch(/Channel not found: General/) + expect(microsoftTeamsConnector.isListingScopeUnavailableError!(error)).toBe(true) + }) + + it('keeps any other listing failure retryable', async () => { + mockGraph({ [CHANNELS_URL]: { status: 500, body: {} } }) + + const error = await listingError() + + expect(error).toBeInstanceOf(Error) + expect(microsoftTeamsConnector.isListingScopeUnavailableError!(error)).toBe(false) + }) +}) + +describe('microsoft teams per-member listing of several channels', () => { + const messagesUrl = (channelId: string) => + `${GRAPH}/teams/${TEAM_ID}/channels/${channelId}/messages?$top=50&$expand=replies` + + function teamsMessage(id: string) { + return { + id, + messageType: 'message', + createdDateTime: '2026-01-01T00:00:00Z', + from: { user: { id: 'u1', displayName: 'Ada' } }, + body: { contentType: 'text', content: `hello from ${id}` }, + } + } + + /** General is readable, Private answers 403 on its messages, Secret is not listed at all. */ + function mockChannels() { + mockGraph({ + [CHANNELS_URL]: { + body: { + value: [ + { id: 'c1', displayName: 'General' }, + { id: 'c2', displayName: 'Private' }, + ], + }, + }, + [messagesUrl('c1')]: { body: { value: [teamsMessage('m1')] } }, + [messagesUrl('c2')]: { status: 403, body: {} }, + }) + } + + beforeEach(() => { + vi.clearAllMocks() + }) + + it('skips the channels the member cannot reach and keeps the rest', async () => { + mockChannels() + + const result = await microsoftTeamsConnector.listDocuments( + 'token', + { teamId: TEAM_ID, channel: ['General', 'Private', 'Secret'] }, + undefined, + { ...PER_MEMBER_LISTING_CONTEXT } + ) + + expect(result.documents.map((doc) => doc.externalId)).toEqual(['c1']) + expect(result.hasMore).toBe(false) + }) + + it('still fails a shared listing when one of several channels is unreachable', async () => { + mockChannels() + + const error = await microsoftTeamsConnector + .listDocuments('token', { teamId: TEAM_ID, channel: ['General', 'Private'] }, undefined, {}) + .catch((e: unknown) => e) + + expect(error).toBeInstanceOf(Error) + expect(microsoftTeamsConnector.isListingScopeUnavailableError!(error)).toBe(true) + }) + + it('reads a sole unreachable channel as the whole scope', async () => { + mockChannels() + + const error = await microsoftTeamsConnector + .listDocuments('token', { teamId: TEAM_ID, channel: 'Private' }, undefined, { + ...PER_MEMBER_LISTING_CONTEXT, + }) + .catch((e: unknown) => e) + + expect(error).toBeInstanceOf(Error) + expect(microsoftTeamsConnector.isListingScopeUnavailableError!(error)).toBe(true) + }) +}) diff --git a/apps/sim/connectors/microsoft-teams/microsoft-teams.ts b/apps/sim/connectors/microsoft-teams/microsoft-teams.ts index f63ec5c0a49..1ae19ae97ef 100644 --- a/apps/sim/connectors/microsoft-teams/microsoft-teams.ts +++ b/apps/sim/connectors/microsoft-teams/microsoft-teams.ts @@ -7,8 +7,11 @@ import { } from '@/connectors/microsoft-teams/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { + ConnectorListingScopeUnavailableError, computeContentHash, htmlToPlainText, + isListingScopeUnavailableError, + isPerMemberListing, parseMultiValue, parseTagDate, } from '@/connectors/utils' @@ -310,14 +313,73 @@ async function resolveChannel( return null } +/** + * Graph answers 403 for a team or private channel the caller is not a member + * of and 404 for one it will not show them; a channel the caller's channel + * list does not resolve is reported the same way. + */ +function isChannelScopeUnavailableError(error: unknown): boolean { + return ( + isListingScopeUnavailableError(error) || + (error instanceof GraphApiError && (error.status === 403 || error.status === 404)) + ) +} + +/** Lists one configured channel as a document, or null when it holds no messages. */ +async function listChannel( + accessToken: string, + teamId: string, + channelInput: string, + maxMessages: number +): Promise { + const channel = await resolveChannel(accessToken, teamId, channelInput) + if (!channel) { + throw new ConnectorListingScopeUnavailableError(`Channel not found: ${channelInput}`, 404) + } + + const { threads, messageCount, lastActivityTs } = await fetchChannelMessages( + accessToken, + teamId, + channel.id, + maxMessages + ) + + const content = formatMessages(threads) + if (!content.trim()) { + logger.info(`No messages found in channel: ${channel.displayName}`) + return null + } + + const contentHash = await computeContentHash(content) + + const sourceUrl = `https://teams.microsoft.com/l/channel/${encodeURIComponent(channel.id)}/${encodeURIComponent(channel.displayName)}?groupId=${encodeURIComponent(teamId)}` + + return { + externalId: channel.id, + title: channel.displayName, + content, + mimeType: 'text/plain', + sourceUrl, + contentHash, + metadata: { + channelName: channel.displayName, + messageCount, + lastActivity: lastActivityTs || undefined, + description: channel.description || undefined, + }, + } +} + export const microsoftTeamsConnector: ConnectorConfig = { ...microsoftTeamsConnectorMeta, + isListingScopeUnavailableError: isChannelScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, _cursor?: string, - _syncContext?: Record + syncContext?: Record ): Promise => { const teamId = sourceConfig.teamId as string const channelInputs = parseMultiValue(sourceConfig.channel) @@ -339,42 +401,32 @@ export const microsoftTeamsConnector: ConnectorConfig = { const documents: ExternalDocument[] = [] for (const channelInput of channelInputs) { - const channel = await resolveChannel(accessToken, teamId, channelInput) - if (!channel) { - throw new Error(`Channel not found: ${channelInput}`) - } - - const { threads, messageCount, lastActivityTs } = await fetchChannelMessages( - accessToken, - teamId, - channel.id, - maxMessages - ) - - const content = formatMessages(threads) - if (!content.trim()) { - logger.info(`No messages found in channel: ${channel.displayName}`) - continue + let document: ExternalDocument | null + try { + document = await listChannel(accessToken, teamId, channelInput, maxMessages) + } catch (error) { + /** + * One of several channels a member cannot reach is absent from their + * listing, not the end of it: move on to the next channel so the rest + * of their access survives. A sole unreachable channel is the whole + * scope, which the members-mode crawl reads as a complete listing of + * nothing, and a shared credential still fails the sync rather than + * silently dropping the channel. + */ + if ( + channelInputs.length > 1 && + isPerMemberListing(syncContext) && + isChannelScopeUnavailableError(error) + ) { + logger.warn('Skipping a Microsoft Teams channel the member cannot reach', { + channel: channelInput, + error: getErrorMessage(error), + }) + continue + } + throw error } - - const contentHash = await computeContentHash(content) - - const sourceUrl = `https://teams.microsoft.com/l/channel/${encodeURIComponent(channel.id)}/${encodeURIComponent(channel.displayName)}?groupId=${encodeURIComponent(teamId)}` - - documents.push({ - externalId: channel.id, - title: channel.displayName, - content, - mimeType: 'text/plain', - sourceUrl, - contentHash, - metadata: { - channelName: channel.displayName, - messageCount, - lastActivity: lastActivityTs || undefined, - description: channel.description || undefined, - }, - }) + if (document) documents.push(document) } // All selected channels are emitted in a single page; no pagination needed diff --git a/apps/sim/connectors/monday/meta.ts b/apps/sim/connectors/monday/meta.ts index ab9ffc882e1..00d01149e2e 100644 --- a/apps/sim/connectors/monday/meta.ts +++ b/apps/sim/connectors/monday/meta.ts @@ -14,6 +14,7 @@ export const mondayConnectorMeta: ConnectorMeta = { requiredScopes: ['boards:read', 'updates:read', 'me:read'], }, + permissionScopedListing: { capFieldIds: ['maxItems'] }, configFields: [ { id: 'boardSelector', diff --git a/apps/sim/connectors/monday/monday.test.ts b/apps/sim/connectors/monday/monday.test.ts index 525523704b7..7ab350c5cff 100644 --- a/apps/sim/connectors/monday/monday.test.ts +++ b/apps/sim/connectors/monday/monday.test.ts @@ -160,6 +160,80 @@ describe('monday listDocuments', () => { }) }) +describe('monday listDocuments with configured boards the caller cannot reach', () => { + it('reports the sole configured board coming back absent as the scope being unavailable', async () => { + mockMonday([{ body: { data: { boards: [] } } }]) + const syncContext: Record = {} + + const error = await mondayConnector + .listDocuments('token', { boardIds: '1' }, undefined, syncContext) + .catch((caught: unknown) => caught) + + expect(mondayConnector.isListingScopeUnavailableError?.(error)).toBe(true) + }) + + it('skips only the unreachable board when another configured board is reachable', async () => { + mockMonday([ + { body: { data: { boards: [] } } }, + { + body: { + data: { + boards: [ + { id: '2', name: 'Board Two', items_page: { cursor: null, items: [item('a')] } }, + ], + }, + }, + }, + ]) + const syncContext: Record = {} + + const first = await mondayConnector.listDocuments( + 'token', + { boardIds: '1,2' }, + undefined, + syncContext + ) + expect(first.documents).toHaveLength(0) + expect(first.hasMore).toBe(true) + + const second = await mondayConnector.listDocuments( + 'token', + { boardIds: '1,2' }, + first.nextCursor, + syncContext + ) + expect(second.documents.map((doc) => doc.externalId)).toEqual(['a']) + expect(second.hasMore).toBe(false) + }) + + it('reports every configured board coming back absent once the last one is walked', async () => { + mockMonday([{ body: { data: { boards: [] } } }, { body: { data: { boards: [] } } }]) + const syncContext: Record = {} + + const first = await mondayConnector.listDocuments( + 'token', + { boardIds: '1,2' }, + undefined, + syncContext + ) + expect(first.hasMore).toBe(true) + + const error = await mondayConnector + .listDocuments('token', { boardIds: '1,2' }, first.nextCursor, syncContext) + .catch((caught: unknown) => caught) + expect(mondayConnector.isListingScopeUnavailableError?.(error)).toBe(true) + }) + + it('does not doubt an enumerated board list, which only ever holds reachable boards', async () => { + mockMonday([{ body: { data: { boards: [] } } }]) + const syncContext: Record = {} + + const result = await mondayConnector.listDocuments('token', {}, undefined, syncContext) + expect(result.documents).toHaveLength(0) + expect(result.hasMore).toBe(false) + }) +}) + describe('monday content extraction', () => { it('falls back to display_value for columns that do not populate text', async () => { mockMonday([ diff --git a/apps/sim/connectors/monday/monday.ts b/apps/sim/connectors/monday/monday.ts index 87050cec2e1..0a50a7819e5 100644 --- a/apps/sim/connectors/monday/monday.ts +++ b/apps/sim/connectors/monday/monday.ts @@ -5,7 +5,12 @@ import { backoffWithJitter } from '@sim/utils/retry' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { mondayConnectorMeta } from '@/connectors/monday/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { parseMultiValue, parseTagDate } from '@/connectors/utils' +import { + ConnectorListingScopeUnavailableError, + isListingScopeUnavailableError, + parseMultiValue, + parseTagDate, +} from '@/connectors/utils' import { MONDAY_API_URL, mondayHeaders } from '@/tools/monday/utils' const logger = createLogger('MondayConnector') @@ -449,6 +454,8 @@ async function resolveBoardIds( export const mondayConnector: ConnectorConfig = { ...mondayConnectorMeta, + isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, @@ -505,6 +512,15 @@ export const mondayConnector: ConnectorConfig = { { ids: [board.id], limit: pageLimit } ) itemsPage = data.boards?.[0]?.items_page ?? null + /** + * `boards(ids:)` filters to the boards the token can see, so a configured + * board the caller cannot reach comes back absent rather than as an error. + * Counted so that a listing which reached none of its configured boards + * can say so below instead of passing as a board that happens to be empty. + */ + if (!data.boards?.length && syncContext) { + syncContext.unreachableBoardCount = ((syncContext.unreachableBoardCount as number) ?? 0) + 1 + } } const items = itemsPage?.items ?? [] @@ -558,6 +574,24 @@ export const mondayConnector: ConnectorConfig = { hasMore = true } + /** + * The caller reached none of the boards this connector is configured for. + * That is the configured scope being unavailable to them — under a member's + * own token a complete listing of nothing, so their access is withdrawn — + * not a set of boards that all happen to be empty. Enumerated boards need no + * such check: the enumeration only ever returns boards the token can see. + */ + if ( + !hasMore && + parseMultiValue(sourceConfig.boardIds).length > 0 && + syncContext?.unreachableBoardCount === boards.length + ) { + throw new ConnectorListingScopeUnavailableError( + `monday.com returned none of the configured boards (${boards.map((b) => b.id).join(', ')}); the account cannot reach them`, + 404 + ) + } + return { documents, nextCursor, hasMore } }, diff --git a/apps/sim/connectors/onedrive/meta.ts b/apps/sim/connectors/onedrive/meta.ts index 4270ad6ddc1..b6dddae1c77 100644 --- a/apps/sim/connectors/onedrive/meta.ts +++ b/apps/sim/connectors/onedrive/meta.ts @@ -10,6 +10,7 @@ export const onedriveConnectorMeta: ConnectorMeta = { auth: { mode: 'oauth', provider: 'onedrive', requiredScopes: ['Files.Read'] }, + permissionScopedListing: { capFieldIds: ['maxFiles'] }, configFields: [ { id: 'folderPath', diff --git a/apps/sim/connectors/onedrive/onedrive.test.ts b/apps/sim/connectors/onedrive/onedrive.test.ts index b086b60f058..7ce1f708fd7 100644 --- a/apps/sim/connectors/onedrive/onedrive.test.ts +++ b/apps/sim/connectors/onedrive/onedrive.test.ts @@ -16,6 +16,7 @@ import { onedriveConnector } from '@/connectors/onedrive/onedrive' import { encodeMicrosoftGraphTraversalCursor, MICROSOFT_GRAPH_MAX_PENDING_FOLDERS, + PER_MEMBER_LISTING_CONTEXT, } from '@/connectors/utils' const GRAPH = 'https://graph.microsoft.com/v1.0' @@ -412,3 +413,71 @@ describe('onedrive getDocument', () => { }) }) }) + +describe('onedrive listing scope', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it.each([403, 404])( + 'reads a %s on the configured folder as a scope the caller cannot reach', + async (status) => { + mockGraph({ [ROOT_URL]: { status, body: {} } }) + + const error = await onedriveConnector.listDocuments('token', {}).catch((e: unknown) => e) + + expect(error).toBeInstanceOf(Error) + expect(onedriveConnector.isListingScopeUnavailableError!(error)).toBe(true) + } + ) + + it('skips a subfolder the member cannot reach and keeps their listing complete', async () => { + mockGraph({ + [ROOT_URL]: { + body: { value: [file('f1', 'a.txt'), folder('open', 'open'), folder('locked', 'locked')] }, + }, + [childrenUrl('locked')]: { status: 403, body: {} }, + [childrenUrl('open')]: { body: { value: [file('f2', 'b.md')] } }, + }) + const syncContext: Record = { ...PER_MEMBER_LISTING_CONTEXT } + + const result = await onedriveConnector.listDocuments('token', {}, undefined, syncContext) + + expect(result.documents.map((d) => d.externalId)).toEqual(['f1', 'f2']) + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('still fails a shared listing on a subfolder it cannot reach', async () => { + mockGraph({ + [ROOT_URL]: { body: { value: [file('f1', 'a.txt'), folder('locked', 'locked')] } }, + }) + + const error = await onedriveConnector + .listDocuments('token', {}, undefined, {}) + .catch((e: unknown) => e) + + expect(error).toBeInstanceOf(Error) + expect(onedriveConnector.isListingScopeUnavailableError!(error)).toBe(true) + }) + + it('reads an unreachable root as the whole scope under a per-member listing', async () => { + mockGraph({ [ROOT_URL]: { status: 403, body: {} } }) + + const error = await onedriveConnector + .listDocuments('token', {}, undefined, { ...PER_MEMBER_LISTING_CONTEXT }) + .catch((e: unknown) => e) + + expect(error).toBeInstanceOf(Error) + expect(onedriveConnector.isListingScopeUnavailableError!(error)).toBe(true) + }) + + it('keeps any other listing failure retryable', async () => { + mockGraph({ [ROOT_URL]: { status: 500, body: {} } }) + + const error = await onedriveConnector.listDocuments('token', {}).catch((e: unknown) => e) + + expect(error).toBeInstanceOf(Error) + expect(onedriveConnector.isListingScopeUnavailableError!(error)).toBe(false) + }) +}) diff --git a/apps/sim/connectors/onedrive/onedrive.ts b/apps/sim/connectors/onedrive/onedrive.ts index bf1c6196b3a..dced987dbaf 100644 --- a/apps/sim/connectors/onedrive/onedrive.ts +++ b/apps/sim/connectors/onedrive/onedrive.ts @@ -18,10 +18,13 @@ import { extractConnectorText, hasIndexablePayload, isIndexableConnectorFile, + isListingScopeUnavailableError, isMicrosoftGraphDriveItem, + isSkippableMicrosoftGraphFolderError, isSkippedDocument, type MicrosoftGraphTraversalState, markSkipped, + microsoftGraphListingError, parseMicrosoftGraphDriveItemList, parseOptionalUnlimitedSafeInteger, parseTagDate, @@ -218,6 +221,8 @@ function decodeCursor(cursor: string): OneDriveTraversalState { export const onedriveConnector: ConnectorConfig = { ...onedriveConnectorMeta, + isListingScopeUnavailableError: isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, @@ -263,7 +268,20 @@ export const onedriveConnector: ConnectorConfig = { status: response.status, error: errorText, }) - throw new Error(`Failed to list OneDrive files: ${response.status}`) + const error = microsoftGraphListingError('Failed to list OneDrive files', response.status) + const isRootFolder = state.currentFolder === undefined + if (!isSkippableMicrosoftGraphFolderError(error, syncContext, isRootFolder)) throw error + logger.warn('Skipping a OneDrive folder the member cannot reach', { + folderId: state.currentFolder, + status: response.status, + }) + if (state.folderStack.length === 0) { + done = true + break + } + state.currentFolder = state.folderStack.pop()! + state.nextLink = undefined + continue } const data = parseMicrosoftGraphDriveItemList(await response.json(), 'OneDrive') diff --git a/apps/sim/connectors/outlook/meta.ts b/apps/sim/connectors/outlook/meta.ts index 379961692d5..96273c90d49 100644 --- a/apps/sim/connectors/outlook/meta.ts +++ b/apps/sim/connectors/outlook/meta.ts @@ -16,6 +16,7 @@ export const outlookConnectorMeta: ConnectorMeta = { requiredScopes: ['Mail.Read'], }, + permissionScopedListing: { capFieldIds: ['maxConversations'] }, configFields: [ { id: 'folderSelector', diff --git a/apps/sim/connectors/outlook/outlook.test.ts b/apps/sim/connectors/outlook/outlook.test.ts index e2258c5c433..7452c2969c3 100644 --- a/apps/sim/connectors/outlook/outlook.test.ts +++ b/apps/sim/connectors/outlook/outlook.test.ts @@ -2,6 +2,7 @@ * @vitest-environment node */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { DEFAULT_MAX_CONVERSATIONS } from '@/connectors/outlook/meta' import { DELETED_ITEMS_FOLDER, isAllMailSync, @@ -470,3 +471,106 @@ describe('getDocument folder exclusion', () => { expect(folderCalls).toHaveLength(2) }) }) + +describe('listDocuments conversation cap', () => { + function inboxMessagesRoute(count: number): [RegExp, () => Response] { + return [ + /\/me\/mailFolders\/inbox\/messages\?/, + () => + jsonResponse({ + value: Array.from({ length: count }, (_, index) => + message({ id: `m${index}`, conversationId: `conv-${index}`, parentFolderId: INBOX_ID }) + ), + }), + ] + } + + it('caps the listing at maxConversations and flags it capped', async () => { + routeFetch([inboxMessagesRoute(3)]) + + const syncContext: Record = {} + const result = await outlookConnector.listDocuments( + 'token', + { folder: 'inbox', maxConversations: '2' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(2) + expect(syncContext.listingCapped).toBe(true) + }) + + it('keeps the default cap when the field holds only whitespace', async () => { + routeFetch([inboxMessagesRoute(DEFAULT_MAX_CONVERSATIONS + 1)]) + + const syncContext: Record = {} + const result = await outlookConnector.listDocuments( + 'token', + { folder: 'inbox', maxConversations: ' ' }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(DEFAULT_MAX_CONVERSATIONS) + expect(syncContext.listingCapped).toBe(true) + }) + + it('lists every conversation when the cap is 0', async () => { + routeFetch([inboxMessagesRoute(3)]) + + const syncContext: Record = {} + const result = await outlookConnector.listDocuments( + 'token', + { folder: 'inbox', maxConversations: 0 }, + undefined, + syncContext + ) + + expect(result.documents).toHaveLength(3) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('reads a folder Graph cannot find as a scope the caller cannot reach', async () => { + routeFetch([[/\/me\/mailFolders\/.*\/messages\?/, () => jsonResponse({}, { status: 404 })]]) + + const error = await outlookConnector + .listDocuments('token', { folder: 'missing-folder-id' }) + .catch((e: unknown) => e) + + expect(error).toBeInstanceOf(Error) + expect(outlookConnector.isListingScopeUnavailableError!(error)).toBe(true) + }) + + it('keeps any other listing failure retryable', async () => { + routeFetch([[/\/me\/mailFolders\/inbox\/messages\?/, () => jsonResponse({}, { status: 500 })]]) + + const error = await outlookConnector + .listDocuments('token', { folder: 'inbox' }) + .catch((e: unknown) => e) + + expect(error).toBeInstanceOf(Error) + expect(outlookConnector.isListingScopeUnavailableError!(error)).toBe(false) + }) +}) + +describe('validateConfig conversation cap', () => { + it('accepts 0 as unlimited', async () => { + routeFetch([[/\/me\/mailFolders\/inbox\/messages\?/, () => jsonResponse({ value: [] })]]) + + await expect( + outlookConnector.validateConfig!('token', { folder: 'inbox', maxConversations: 0 }) + ).resolves.toEqual({ valid: true }) + }) + + it('rejects a fractional cap without calling Graph', async () => { + routeFetch([]) + + await expect( + outlookConnector.validateConfig!('token', { folder: 'inbox', maxConversations: '1.5' }) + ).resolves.toEqual({ + valid: false, + error: 'Max conversations must be a positive safe integer, or 0 for unlimited', + }) + expect(fetchMock).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/connectors/outlook/outlook.ts b/apps/sim/connectors/outlook/outlook.ts index 25661fb0cda..78f183883d6 100644 --- a/apps/sim/connectors/outlook/outlook.ts +++ b/apps/sim/connectors/outlook/outlook.ts @@ -3,7 +3,13 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { DEFAULT_MAX_CONVERSATIONS, outlookConnectorMeta } from '@/connectors/outlook/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { htmlToPlainText, parseTagDate } from '@/connectors/utils' +import { + htmlToPlainText, + isListingScopeUnavailableError, + listingRequestError, + parseDefaultedUnlimitedSafeInteger, + parseTagDate, +} from '@/connectors/utils' const logger = createLogger('OutlookConnector') @@ -602,19 +608,30 @@ function formatConversation( } } +/** + * The conversation cap. A blank field keeps the default; 0 lifts the cap so a + * per-member listing is complete. + */ +function parseMaxConversations(value: unknown): number { + return parseDefaultedUnlimitedSafeInteger( + value, + DEFAULT_MAX_CONVERSATIONS, + 'Max conversations must be a positive safe integer, or 0 for unlimited' + ) +} + export const outlookConnector: ConnectorConfig = { ...outlookConnectorMeta, + isListingScopeUnavailableError: isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, cursor?: string, syncContext?: Record ): Promise => { - /** `validateConfig` rejects a non-positive value, so anything else here is drift. */ - const parsedMax = Number(sourceConfig.maxConversations) - const maxConversations = - Number.isFinite(parsedMax) && parsedMax > 0 ? parsedMax : DEFAULT_MAX_CONVERSATIONS + const maxConversations = parseMaxConversations(sourceConfig.maxConversations) // Initialize accumulator in syncContext if (syncContext && !syncContext._conversations) { @@ -648,7 +665,7 @@ export const outlookConnector: ConnectorConfig = { status: response.status, error: errorText, }) - throw new Error(`Failed to fetch Outlook messages: ${response.status}`) + throw listingRequestError('Failed to fetch Outlook messages', response.status) } const data = await response.json() @@ -749,11 +766,12 @@ export const outlookConnector: ConnectorConfig = { }) /** - * Limit to `maxConversations`. Dropping the overflow makes the listing an - * incomplete view of the mailbox, so it is flagged as capped — otherwise - * reconciliation would hard-delete every conversation past the cap. + * Limit to `maxConversations` when one is set. Dropping the overflow makes + * the listing an incomplete view of the mailbox, so it is flagged as capped — + * otherwise reconciliation would hard-delete every conversation past the cap. */ - const limited = conversationEntries.slice(0, maxConversations) + const limited = + maxConversations > 0 ? conversationEntries.slice(0, maxConversations) : conversationEntries if (conversationEntries.length > limited.length && syncContext) { syncContext.listingCapped = true } @@ -911,13 +929,10 @@ export const outlookConnector: ConnectorConfig = { accessToken: string, sourceConfig: Record ): Promise<{ valid: boolean; error?: string }> => { - const maxConversations = sourceConfig.maxConversations as string | undefined - - if ( - maxConversations && - (Number.isNaN(Number(maxConversations)) || Number(maxConversations) <= 0) - ) { - return { valid: false, error: 'Max conversations must be a positive number' } + try { + parseMaxConversations(sourceConfig.maxConversations) + } catch (error) { + return { valid: false, error: toError(error).message } } try { diff --git a/apps/sim/connectors/permission-scoped-listing.test.ts b/apps/sim/connectors/permission-scoped-listing.test.ts new file mode 100644 index 00000000000..1e66c9b5f33 --- /dev/null +++ b/apps/sim/connectors/permission-scoped-listing.test.ts @@ -0,0 +1,91 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { getManagedOAuthConnectorPolicy } from '@/lib/auth/connectors/managed-oauth' +import { + getCredentialGroupProviderService, + getCredentialGroupStandardOAuthProviderFromProviderId, +} from '@/lib/credential-groups/providers' +import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' + +const permissionScoped = Object.values(CONNECTOR_META_REGISTRY).filter( + (meta) => meta.permissionScopedListing !== undefined +) + +/** + * A connector that crawls per member mints each member's token from a + * Credential Group option, and that option requests exactly the scopes its + * provider's managed policy defines. The connector's own read scopes must fit + * inside them, or every member would be refused at sync time with no way to + * fix it from the connector's settings. + */ +describe('permission-scoped connector listings', () => { + it('covers the connectors that crawl per member', () => { + expect(permissionScoped.map((meta) => meta.id).sort()).toEqual([ + 'airtable', + 'asana', + 'bitbucket', + 'box', + 'clickup', + 'confluence', + 'docusign', + 'dropbox', + 'gmail', + 'google_calendar', + 'google_chat', + 'google_docs', + 'google_drive', + 'google_forms', + 'google_meet', + 'google_sheets', + 'google_slides', + 'jira', + 'jsm', + 'linear', + 'microsoft_excel', + 'microsoft_teams', + 'monday', + 'onedrive', + 'outlook', + 'salesforce', + 'sharepoint', + 'zoom', + ]) + }) + + it.each(permissionScoped.map((meta) => [meta.id, meta] as const))( + '%s authenticates through a managed OAuth provider whose option scopes cover its read scopes', + (_id, meta) => { + expect(meta.auth.mode).toBe('oauth') + if (meta.auth.mode !== 'oauth') return + + const policy = getManagedOAuthConnectorPolicy(meta.auth.provider) + expect(policy).toBeDefined() + if (!policy) return + + const groupProvider = getCredentialGroupStandardOAuthProviderFromProviderId( + meta.auth.provider + ) + expect(groupProvider).toBeDefined() + + const optionScopes = [ + ...new Set([ + ...getCredentialGroupProviderService(groupProvider).scopes, + ...policy.additionalScopes, + ]), + ] + expect(policy.hasRequiredScopes(optionScopes, meta.auth.requiredScopes ?? [])).toBe(true) + } + ) + + it.each(permissionScoped.map((meta) => [meta.id, meta] as const))( + '%s names only real config fields as listing caps', + (_id, meta) => { + const fieldIds = new Set(meta.configFields.map((field) => field.id)) + for (const capFieldId of meta.permissionScopedListing?.capFieldIds ?? []) { + expect(fieldIds.has(capFieldId)).toBe(true) + } + } + ) +}) diff --git a/apps/sim/connectors/salesforce/meta.ts b/apps/sim/connectors/salesforce/meta.ts index bf9b4e0fa97..0340bad70dc 100644 --- a/apps/sim/connectors/salesforce/meta.ts +++ b/apps/sim/connectors/salesforce/meta.ts @@ -21,6 +21,9 @@ export const salesforceConnectorMeta: ConnectorMeta = { requiredScopes: ['api', 'refresh_token', 'openid'], }, + /** Every synced object carries `LastModifiedDate`, which the listing filters on. */ + supportsIncrementalSync: true, + permissionScopedListing: { capFieldIds: ['maxRecords'] }, configFields: [ { id: 'objectType', diff --git a/apps/sim/connectors/salesforce/salesforce.test.ts b/apps/sim/connectors/salesforce/salesforce.test.ts new file mode 100644 index 00000000000..6bf56c4ceba --- /dev/null +++ b/apps/sim/connectors/salesforce/salesforce.test.ts @@ -0,0 +1,124 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() })) + +vi.mock('@/lib/knowledge/documents/utils', () => ({ + fetchWithRetry: mockFetchWithRetry, + VALIDATE_RETRY_OPTIONS: {}, +})) +vi.mock('@/components/icons', () => ({ SalesforceIcon: () => null })) + +import { salesforceConnector } from '@/connectors/salesforce/salesforce' + +const INSTANCE_URL = 'https://org.example.com/services/data/v62.0/' + +/** Answers every query with `status` and `body`, recording the requested URLs. */ +function mockQuery(status: number, body: unknown) { + const urls: string[] = [] + mockFetchWithRetry.mockImplementation(async (url: string) => { + urls.push(url) + return { + ok: status >= 200 && status < 300, + status, + json: async () => body, + text: async () => JSON.stringify(body), + } as unknown as Response + }) + return urls +} + +function syncContext(): Record { + return { instanceUrl: INSTANCE_URL } +} + +async function listingError(sourceConfig: Record): Promise { + return salesforceConnector + .listDocuments('token', sourceConfig, undefined, syncContext()) + .catch((caught: unknown) => caught) +} + +function soqlOf(url: string): string { + return decodeURIComponent(new URL(url).searchParams.get('q') ?? '') +} + +beforeEach(() => { + vi.clearAllMocks() +}) + +describe('salesforce listing scope classification', () => { + it('treats an object the caller may not read (400 INVALID_TYPE) as the scope being unavailable', async () => { + mockQuery(400, [ + { message: "sObject type 'Case' is not supported.", errorCode: 'INVALID_TYPE' }, + ]) + const error = await listingError({ objectType: 'Case' }) + expect(salesforceConnector.isListingScopeUnavailableError?.(error)).toBe(true) + }) + + it('treats an explicit denial (403 INSUFFICIENT_ACCESS) the same way', async () => { + mockQuery(403, [{ message: 'denied', errorCode: 'INSUFFICIENT_ACCESS_OR_READONLY' }]) + const error = await listingError({ objectType: 'Account' }) + expect(salesforceConnector.isListingScopeUnavailableError?.(error)).toBe(true) + }) + + it('leaves a malformed query and server faults for the sync engines to retry', async () => { + mockQuery(400, [{ message: 'unexpected token', errorCode: 'MALFORMED_QUERY' }]) + const malformed = await listingError({ objectType: 'Account' }) + expect(malformed).toBeInstanceOf(Error) + expect(salesforceConnector.isListingScopeUnavailableError?.(malformed)).toBe(false) + + mockQuery(500, 'Internal Server Error') + const fault = await listingError({ objectType: 'Account' }) + expect(fault).toBeInstanceOf(Error) + expect(salesforceConnector.isListingScopeUnavailableError?.(fault)).toBe(false) + }) +}) + +describe('salesforce incremental listing', () => { + it('advertises incremental sync', () => { + expect(salesforceConnector.supportsIncrementalSync).toBe(true) + }) + + it('lists the whole object when no watermark is given', async () => { + const urls = mockQuery(200, { records: [] }) + await salesforceConnector.listDocuments( + 'token', + { objectType: 'Case' }, + undefined, + syncContext() + ) + expect(soqlOf(urls[0])).toBe( + 'SELECT Id,Subject,Description,Status,LastModifiedDate,CaseNumber FROM Case ORDER BY LastModifiedDate DESC' + ) + }) + + it('filters on LastModifiedDate with an unquoted UTC literal after a watermark', async () => { + const urls = mockQuery(200, { records: [] }) + await salesforceConnector.listDocuments( + 'token', + { objectType: 'Case' }, + undefined, + syncContext(), + new Date('2026-09-01T12:34:56.789Z') + ) + expect(soqlOf(urls[0])).toContain( + ' FROM Case WHERE LastModifiedDate >= 2026-09-01T12:34:56Z ORDER BY' + ) + }) + + it('appends the watermark to the mandatory Knowledge Article filters', async () => { + const urls = mockQuery(200, { records: [] }) + await salesforceConnector.listDocuments( + 'token', + { objectType: 'KnowledgeArticleVersion' }, + undefined, + syncContext(), + new Date('2026-09-01T00:00:00Z') + ) + expect(soqlOf(urls[0])).toContain( + "WHERE PublishStatus='Online' AND IsLatestVersion=true AND Language='en_US' AND LastModifiedDate >= 2026-09-01T00:00:00Z ORDER BY" + ) + }) +}) diff --git a/apps/sim/connectors/salesforce/salesforce.ts b/apps/sim/connectors/salesforce/salesforce.ts index ef075fc004c..522e290d834 100644 --- a/apps/sim/connectors/salesforce/salesforce.ts +++ b/apps/sim/connectors/salesforce/salesforce.ts @@ -4,7 +4,12 @@ import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/document import { SALESFORCE_LOGIN_HOSTS } from '@/lib/oauth/salesforce' import { salesforceConnectorMeta } from '@/connectors/salesforce/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' -import { htmlToPlainText, parseTagDate } from '@/connectors/utils' +import { + htmlToPlainText, + isListingScopeUnavailableError, + listingRequestError, + parseTagDate, +} from '@/connectors/utils' const logger = createLogger('SalesforceConnector') @@ -84,9 +89,52 @@ const OBJECT_FIELDS: Record = { * user-selectable locale — rather than relying on that hedge holding for the * abstract KnowledgeArticleVersion view. */ -function buildWhereClause(objectType: string, language: string): string { - if (objectType !== 'KnowledgeArticleVersion') return '' - return ` WHERE PublishStatus='Online' AND IsLatestVersion=true AND Language='${language}'` +function buildWhereClause(objectType: string, language: string, lastSyncAt?: Date): string { + const conditions: string[] = [] + if (objectType === 'KnowledgeArticleVersion') { + conditions.push("PublishStatus='Online'", 'IsLatestVersion=true', `Language='${language}'`) + } + if (lastSyncAt) conditions.push(`LastModifiedDate >= ${toSoqlDateTime(lastSyncAt)}`) + return conditions.length > 0 ? ` WHERE ${conditions.join(' AND ')}` : '' +} + +/** + * A SOQL dateTime literal: ISO 8601 in UTC, unquoted, and without the + * fractional seconds SOQL does not accept. + */ +function toSoqlDateTime(date: Date): string { + return date.toISOString().replace(/\.\d{3}Z$/, 'Z') +} + +/** The `errorCode` values in a Salesforce REST error body (a JSON array of errors). */ +function parseSalesforceErrorCodes(errorText: string): string[] { + try { + const parsed: unknown = JSON.parse(errorText) + if (!Array.isArray(parsed)) return [] + return parsed.flatMap((entry: unknown) => + typeof entry === 'object' && + entry !== null && + typeof (entry as { errorCode?: unknown }).errorCode === 'string' + ? [(entry as { errorCode: string }).errorCode] + : [] + ) + } catch { + return [] + } +} + +/** + * Whether a failed query means the caller cannot read the configured object at + * all. Salesforce hides an object from a user who may not read it, so the query + * fails with 400 `INVALID_TYPE` rather than returning nothing, and an explicit + * denial is 403 `INSUFFICIENT_ACCESS`; either is a complete listing of nothing + * for that caller, while anything else is a fault the sync engines retry. + */ +function isSalesforceAccessDenied(status: number, errorText: string): boolean { + if (status !== 400 && status !== 403) return false + return parseSalesforceErrorCodes(errorText).some( + (code) => code === 'INVALID_TYPE' || code.startsWith('INSUFFICIENT_ACCESS') + ) } /** @@ -326,11 +374,14 @@ function recordToDocument( export const salesforceConnector: ConnectorConfig = { ...salesforceConnectorMeta, + isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, cursor?: string, - syncContext?: Record + syncContext?: Record, + lastSyncAt?: Date ): Promise => { const objectType = sourceConfig.objectType as string const maxRecords = sourceConfig.maxRecords ? Number(sourceConfig.maxRecords) : 0 @@ -347,7 +398,11 @@ export const salesforceConnector: ConnectorConfig = { if (cursor) { url = `${toOrigin(instanceUrl)}${cursor}` } else { - const whereClause = buildWhereClause(objectType, resolveArticleLanguage(sourceConfig)) + const whereClause = buildWhereClause( + objectType, + resolveArticleLanguage(sourceConfig), + lastSyncAt + ) /** * No SOQL `LIMIT`: it bounds the total result set rather than the batch, * so it would end the sync after a single page. Paging is driven by @@ -359,7 +414,10 @@ export const salesforceConnector: ConnectorConfig = { url = `${instanceUrl}query?q=${encodeURIComponent(soql)}` } - logger.info(`Listing Salesforce ${objectType}`, { cursor: cursor || 'initial' }) + logger.info(`Listing Salesforce ${objectType}`, { + cursor: cursor || 'initial', + incremental: Boolean(lastSyncAt), + }) const response = await fetchWithRetry(url, { method: 'GET', @@ -375,7 +433,11 @@ export const salesforceConnector: ConnectorConfig = { status: response.status, error: errorText, }) - throw new Error(`Failed to query Salesforce ${objectType}: ${response.status}`) + throw listingRequestError( + `Failed to query Salesforce ${objectType}`, + response.status, + isSalesforceAccessDenied(response.status, errorText) + ) } const data = await response.json() diff --git a/apps/sim/connectors/sharepoint/meta.ts b/apps/sim/connectors/sharepoint/meta.ts index 38dbff4e259..ee9d0baee1d 100644 --- a/apps/sim/connectors/sharepoint/meta.ts +++ b/apps/sim/connectors/sharepoint/meta.ts @@ -10,6 +10,7 @@ export const sharepointConnectorMeta: ConnectorMeta = { auth: { mode: 'oauth', provider: 'sharepoint', requiredScopes: ['Sites.Read.All'] }, + permissionScopedListing: { capFieldIds: ['maxFiles'] }, configFields: [ { id: 'siteUrl', diff --git a/apps/sim/connectors/sharepoint/sharepoint.test.ts b/apps/sim/connectors/sharepoint/sharepoint.test.ts index 7871174bdf9..9263246515b 100644 --- a/apps/sim/connectors/sharepoint/sharepoint.test.ts +++ b/apps/sim/connectors/sharepoint/sharepoint.test.ts @@ -22,6 +22,7 @@ import { appendPendingMicrosoftGraphFolders, encodeMicrosoftGraphTraversalCursor, MICROSOFT_GRAPH_MAX_PENDING_FOLDERS, + PER_MEMBER_LISTING_CONTEXT, } from '@/connectors/utils' const GRAPH = 'https://graph.microsoft.com/v1.0' @@ -479,6 +480,52 @@ describe('listDocuments', () => { expect(syncContext.listingCapped).toBeUndefined() }) + it('skips a subfolder the member cannot reach and keeps their listing complete', async () => { + mockGraph({ + ...childrenRoute(DEFAULT_DRIVE_ID, null, [ + file('f1', 'a.txt'), + folder('open', 'Open'), + folder('locked', 'Locked'), + ]), + [`${GRAPH}/drives/${DEFAULT_DRIVE_ID}/items/locked/children?$top=200&$select=${ITEM_SELECT}`]: + { status: 403, body: {} }, + ...childrenRoute(DEFAULT_DRIVE_ID, 'open', [file('f2', 'b.txt')]), + }) + const syncContext = { ...listContext(), ...PER_MEMBER_LISTING_CONTEXT } + + const result = await list(undefined, syncContext) + + expect(result.documents.map((doc) => doc.externalId)).toEqual(['f1', 'f2']) + expect(result.hasMore).toBe(false) + expect(syncContext.listingCapped).toBeUndefined() + }) + + it('still fails a shared listing on a subfolder it cannot reach', async () => { + mockGraph({ + ...childrenRoute(DEFAULT_DRIVE_ID, null, [file('f1', 'a.txt'), folder('locked', 'Locked')]), + }) + + const error = await list(undefined, listContext()).catch((e: unknown) => e) + + expect(error).toBeInstanceOf(Error) + expect(sharepointConnector.isListingScopeUnavailableError!(error)).toBe(true) + }) + + it('reads an unreachable root as the whole scope under a per-member listing', async () => { + mockGraph({ + [`${GRAPH}/drives/${DEFAULT_DRIVE_ID}/root/children?$top=200&$select=${ITEM_SELECT}`]: { + status: 403, + body: {}, + }, + }) + const syncContext = { ...listContext(), ...PER_MEMBER_LISTING_CONTEXT } + + const error = await list(undefined, syncContext).catch((e: unknown) => e) + + expect(error).toBeInstanceOf(Error) + expect(sharepointConnector.isListingScopeUnavailableError!(error)).toBe(true) + }) + it('drains subfolders within a single call instead of one folder per page', async () => { mockGraph({ ...childrenRoute(DEFAULT_DRIVE_ID, null, [file('f1', 'a.txt'), folder('sub', 'Sub')]), @@ -801,3 +848,44 @@ describe('normalizeSegment', () => { expect(normalizeSegment('Reports')).toBe('reports') }) }) + +describe('listing scope', () => { + it.each([403, 404])( + 'reads a %s on the configured site as a scope the caller cannot reach', + async (status) => { + mockGraph({ [`${GRAPH}/sites/${SITE_URL}`]: { status, body: {} } }) + + const error = await sharepointConnector + .listDocuments('token', { siteUrl: SITE_URL }) + .catch((e: unknown) => e) + + expect(error).toBeInstanceOf(Error) + expect(sharepointConnector.isListingScopeUnavailableError!(error)).toBe(true) + } + ) + + it('reads a folder Graph will not show the caller as a scope they cannot reach', async () => { + mockGraph({ + ...defaultDriveRoute, + ...sitesDrivesRoute, + ...rootChildren(DEFAULT_DRIVE_ID, [folder('a', 'Archive')]), + }) + + const error = await resolve('Reports').catch((e: unknown) => e) + + expect(error).toBeInstanceOf(Error) + expect(String(error)).toMatch(/Folder not found: "Reports"/) + expect(sharepointConnector.isListingScopeUnavailableError!(error)).toBe(true) + }) + + it('keeps any other failure retryable', async () => { + mockGraph({ [`${GRAPH}/sites/${SITE_URL}`]: { status: 500, body: {} } }) + + const error = await sharepointConnector + .listDocuments('token', { siteUrl: SITE_URL }) + .catch((e: unknown) => e) + + expect(error).toBeInstanceOf(Error) + expect(sharepointConnector.isListingScopeUnavailableError!(error)).toBe(false) + }) +}) diff --git a/apps/sim/connectors/sharepoint/sharepoint.ts b/apps/sim/connectors/sharepoint/sharepoint.ts index 043fe0d5cf9..27f21bd6fab 100644 --- a/apps/sim/connectors/sharepoint/sharepoint.ts +++ b/apps/sim/connectors/sharepoint/sharepoint.ts @@ -13,16 +13,20 @@ import { assertMicrosoftGraphNextLink, CONNECTOR_MAX_FILE_BYTES, ConnectorFileTooLargeError, + ConnectorListingScopeUnavailableError, connectorFileExtension, decodeMicrosoftGraphTraversalCursor, encodeMicrosoftGraphTraversalCursor, extractConnectorText, hasIndexablePayload, isIndexableConnectorFile, + isListingScopeUnavailableError, isMicrosoftGraphDriveItem, + isSkippableMicrosoftGraphFolderError, isSkippedDocument, type MicrosoftGraphTraversalState, markSkipped, + microsoftGraphListingError, parseMicrosoftGraphDriveItemList, parseOptionalUnlimitedSafeInteger, parseTagDate, @@ -216,8 +220,10 @@ async function resolveSiteId( if (!response.ok) { const errorText = await readBoundedHttpErrorBody(response) - throw new Error( - `Failed to resolve SharePoint site "${siteUrl}": ${response.status} – ${errorText}` + throw microsoftGraphListingError( + `Failed to resolve SharePoint site "${siteUrl}"`, + response.status, + errorText ) } @@ -323,7 +329,7 @@ async function listFolderItems( if (!response.ok) { const errorText = await readBoundedHttpErrorBody(response) - throw new Error(`Failed to list folder items: ${response.status} – ${errorText}`) + throw microsoftGraphListingError('Failed to list folder items', response.status, errorText) } const data = parseMicrosoftGraphDriveItemList(await response.json(), 'SharePoint') @@ -403,7 +409,7 @@ async function getItemByPath( if (response.status === 404) return null if (!response.ok) { - throw new Error(`Failed to resolve folder path: ${response.status}`) + throw microsoftGraphListingError('Failed to resolve folder path', response.status) } return (await response.json()) as DriveItem @@ -427,7 +433,7 @@ async function listChildFolders( const response = await graphGet(url, accessToken, retryOptions) if (!response.ok) { const errorText = await readBoundedHttpErrorBody(response) - throw new Error(`Failed to list folder contents: ${response.status} – ${errorText}`) + throw microsoftGraphListingError('Failed to list folder contents', response.status, errorText) } const rawData: unknown = await response.json() @@ -564,8 +570,9 @@ export async function resolveFolderTarget( retryOptions ) if (!defaultDriveResponse.ok) { - throw new Error( - `Failed to open the default document library for site "${siteUrl}": ${defaultDriveResponse.status}` + throw microsoftGraphListingError( + `Failed to open the default document library for site "${siteUrl}"`, + defaultDriveResponse.status ) } const defaultDrive = (await defaultDriveResponse.json()) as Drive @@ -648,7 +655,8 @@ export async function resolveFolderTarget( ? { id: libraryMatch.id, name: libraryMatch.name || segments[0] } : { id: defaultDrive.id, name: defaultDriveName } - throw new Error( + /** A folder Graph will not show this caller is, for them, a scope of nothing. */ + throw new ConnectorListingScopeUnavailableError( await buildFolderNotFoundMessage( accessToken, reportDrive, @@ -658,7 +666,8 @@ export async function resolveFolderTarget( drives, reportDrive.id === defaultDrive.id, retryOptions - ) + ), + 404 ) } @@ -675,8 +684,10 @@ async function listSiteDrives( const response = await graphGet(url, accessToken, retryOptions) if (!response.ok) { const errorText = await readBoundedHttpErrorBody(response) - throw new Error( - `Failed to list SharePoint document libraries: ${response.status} – ${errorText}` + throw microsoftGraphListingError( + 'Failed to list SharePoint document libraries', + response.status, + errorText ) } const data = parseDriveListResponse(await response.json()) @@ -784,6 +795,8 @@ function decodeCursor(cursor: string): PaginationState { export const sharepointConnector: ConnectorConfig = { ...sharepointConnectorMeta, + isListingScopeUnavailableError: isListingScopeUnavailableError, + listDocuments: async ( accessToken: string, sourceConfig: Record, @@ -853,7 +866,24 @@ export const sharepointConnector: ConnectorConfig = { let cappedWithItemsLeft = false for (let request = 0; request < MAX_LIST_REQUESTS_PER_CALL; request++) { - const data = await listFolderItems(accessToken, driveId, state.currentFolder, state.nextLink) + let data: Awaited> + try { + data = await listFolderItems(accessToken, driveId, state.currentFolder, state.nextLink) + } catch (error) { + const isRootFolder = state.currentFolder === rootFolderId + if (!isSkippableMicrosoftGraphFolderError(error, syncContext, isRootFolder)) throw error + logger.warn('Skipping a SharePoint folder the member cannot reach', { + folderId: state.currentFolder, + error: getErrorMessage(error), + }) + if (state.folderStack.length === 0) { + stopPaging = true + break + } + state.currentFolder = state.folderStack.pop()! + state.nextLink = undefined + continue + } // Separate files and subfolders const subfolders: string[] = [] diff --git a/apps/sim/connectors/types.ts b/apps/sim/connectors/types.ts index cc11ee3fd7a..f462cb29dc3 100644 --- a/apps/sim/connectors/types.ts +++ b/apps/sim/connectors/types.ts @@ -95,6 +95,24 @@ export interface ExternalDocumentList { reconciliationSafe?: boolean } +/** + * One entry of a connector's change feed. A removal is authoritative for the + * caller: the item was deleted, or their account can no longer reach it. + */ +export type ExternalChange = + | { kind: 'upsert'; externalId: string; document: ExternalDocument } + | { kind: 'removed'; externalId: string } + +export interface ExternalChangeList { + changes: ExternalChange[] + /** + * The cursor to continue from: the next page while `hasMore`, otherwise the + * point the drained feed should resume from on the next read. + */ + nextCursor: string + hasMore: boolean +} + export const SYNC_SKIP_REASONS = [ 'connector_unavailable', 'knowledge_base_deleted', @@ -218,6 +236,16 @@ export interface ConnectorMeta { * on the KB for enabled slots, and mapTags output is filtered to only include them. */ tagDefinitions?: ConnectorTagDefinition[] + + /** + * Set when a listing under one person's own token returns exactly the documents + * that person may read, so a knowledge base can crawl the source once per + * enrolled member and take each member's listing as that member's access. + * Names the config fields that cap the listing: a cap would hide part of a + * member's corpus and suppress removals forever, so those fields are refused + * whenever the connector crawls per member. + */ + permissionScopedListing?: { capFieldIds: readonly string[] } } /** @@ -266,6 +294,46 @@ export interface ConnectorConfig extends ConnectorMeta { sourceConfig: Record ) => Promise<{ valid: boolean; error?: string }> + /** + * Whether a listing failure means the caller simply cannot reach the + * configured scope — the folder or space is not shared with them. A + * members-mode crawl treats that as a complete listing of nothing for that + * member rather than an error, so their access is withdrawn instead of + * retried forever. Only meaningful alongside {@link ConnectorMeta.permissionScopedListing}. + */ + isListingScopeUnavailableError?: (error: unknown) => boolean + + /** + * Opens a change feed over the caller's view of the source: the cursor from + * which {@link listChanges} later reports everything they gain, lose, or see + * modified. A members-mode crawl takes it before a full listing so nothing + * that changes during the listing is missed, and then reads the feed on + * every run instead of relisting. Only meaningful alongside + * {@link ConnectorMeta.permissionScopedListing}. + */ + getChangeCursor?: ( + accessToken: string, + sourceConfig: Record, + syncContext?: Record + ) => Promise + + /** + * Reads the change feed from a cursor. An upsert carries the same stub a + * listing would; a removal withdraws the caller's access to the item. + */ + listChanges?: ( + accessToken: string, + sourceConfig: Record, + cursor: string, + syncContext?: Record + ) => Promise + + /** + * Whether a change-feed failure means the cursor has expired, so the feed + * must be reopened from a fresh full listing rather than retried. + */ + isChangeCursorInvalidError?: (error: unknown) => boolean + /** Map source metadata to semantic tag keys (translated to slots by the sync engine) */ mapTags?: (metadata: Record) => Record } diff --git a/apps/sim/connectors/utils.test.ts b/apps/sim/connectors/utils.test.ts index 4fd76920f55..353d715eaa0 100644 --- a/apps/sim/connectors/utils.test.ts +++ b/apps/sim/connectors/utils.test.ts @@ -64,17 +64,21 @@ import { appendPendingMicrosoftGraphFolders, assertMicrosoftGraphNextLink, ConnectorFileTooLargeError, + ConnectorListingScopeUnavailableError, decodeMicrosoftGraphTraversalCursor, encodeMicrosoftGraphTraversalCursor, extractConnectorText, hasIndexablePayload, htmlToPlainText, isIndexableConnectorFile, + isSkippableMicrosoftGraphFolderError, isSkippedDocument, MICROSOFT_GRAPH_MAX_CURSOR_ENCODED_BYTES, MICROSOFT_GRAPH_MAX_ITEM_ID_BYTES, MICROSOFT_GRAPH_MAX_PENDING_FOLDERS, markSkipped, + PER_MEMBER_LISTING_CONTEXT, + parseDefaultedUnlimitedSafeInteger, pipelineParsedMimeType, readBodyWithLimit, sizeLimitSkipReason, @@ -1594,3 +1598,44 @@ describe('hasIndexablePayload', () => { expect(hasIndexablePayload({ content: ' ' })).toBe(false) }) }) + +describe('parseDefaultedUnlimitedSafeInteger', () => { + const ERROR = 'bad cap' + + it.each([undefined, null, '', ' '])('keeps the default for a blank field (%j)', (value) => { + expect(parseDefaultedUnlimitedSafeInteger(value, 500, ERROR)).toBe(500) + }) + + it('reads an explicit 0 as unlimited', () => { + expect(parseDefaultedUnlimitedSafeInteger(0, 500, ERROR)).toBe(0) + expect(parseDefaultedUnlimitedSafeInteger('0', 500, ERROR)).toBe(0) + }) + + it('parses a set cap and rejects a malformed one', () => { + expect(parseDefaultedUnlimitedSafeInteger(' 200 ', 500, ERROR)).toBe(200) + expect(() => parseDefaultedUnlimitedSafeInteger('many', 500, ERROR)).toThrow(ERROR) + expect(() => parseDefaultedUnlimitedSafeInteger(-1, 500, ERROR)).toThrow(ERROR) + }) +}) + +describe('isSkippableMicrosoftGraphFolderError', () => { + const unreachable = new ConnectorListingScopeUnavailableError('folder', 403) + const perMember = { ...PER_MEMBER_LISTING_CONTEXT } + + it('skips an unreachable descendant folder under a per-member listing', () => { + expect(isSkippableMicrosoftGraphFolderError(unreachable, perMember, false)).toBe(true) + }) + + it('never skips the configured root', () => { + expect(isSkippableMicrosoftGraphFolderError(unreachable, perMember, true)).toBe(false) + }) + + it('never skips under a shared credential', () => { + expect(isSkippableMicrosoftGraphFolderError(unreachable, {}, false)).toBe(false) + expect(isSkippableMicrosoftGraphFolderError(unreachable, undefined, false)).toBe(false) + }) + + it('never skips a fault the engine should retry', () => { + expect(isSkippableMicrosoftGraphFolderError(new Error('500'), perMember, false)).toBe(false) + }) +}) diff --git a/apps/sim/connectors/utils.ts b/apps/sim/connectors/utils.ts index 8672c1ecb62..857bac814a7 100644 --- a/apps/sim/connectors/utils.ts +++ b/apps/sim/connectors/utils.ts @@ -56,6 +56,22 @@ export function parseOptionalUnlimitedSafeInteger(value: unknown, errorMessage: return parsed } +/** + * Parses a connector cap that keeps `defaultValue` when the field is blank — + * absent, null, or a string of nothing but whitespace — and otherwise reads + * like {@link parseOptionalUnlimitedSafeInteger}, where 0 lifts the cap. A + * per-member sync writes that explicit 0; a form left empty must not. + */ +export function parseDefaultedUnlimitedSafeInteger( + value: unknown, + defaultValue: number, + errorMessage: string +): number { + if (value === undefined || value === null) return defaultValue + if (typeof value === 'string' && value.trim() === '') return defaultValue + return parseOptionalUnlimitedSafeInteger(value, errorMessage) +} + const MICROSOFT_GRAPH_ORIGIN = 'https://graph.microsoft.com' export interface MicrosoftGraphTraversalState { @@ -695,3 +711,87 @@ export class ConnectorFileTooLargeError extends Error { this.name = 'ConnectorFileTooLargeError' } } + +/** + * A listing failed because the caller cannot reach the configured scope — the + * folder, space, board, or calendar is not shared with them. A members-mode + * crawl treats that as a complete listing of nothing for that member, so + * their access is withdrawn rather than retried forever. + */ +export class ConnectorListingScopeUnavailableError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + this.name = 'ConnectorListingScopeUnavailableError' + } +} + +/** + * The error a listing request throws for a failed response: scope-unavailable + * when the source says the scope does not exist for this caller (404, or + * whatever `scopeUnavailable` recognises in the source's own error body), a + * plain error for anything else, which the sync engines retry with backoff. + */ +export function listingRequestError( + message: string, + status: number, + scopeUnavailable: boolean = status === 404 +): Error { + const described = `${message}: ${status}` + return scopeUnavailable + ? new ConnectorListingScopeUnavailableError(described, status) + : new Error(described) +} + +export function isListingScopeUnavailableError(error: unknown): boolean { + return error instanceof ConnectorListingScopeUnavailableError +} + +/** + * The error a failed Microsoft Graph listing request throws. Graph reports a + * drive, site, or folder the caller cannot reach as 404 (`itemNotFound`) or + * 403 (`accessDenied`); either is a complete listing of nothing for that + * caller, while anything else is a fault the sync engines retry. + */ +export function microsoftGraphListingError( + message: string, + status: number, + detail?: string +): Error { + const described = detail ? `${message}: ${status} – ${detail}` : `${message}: ${status}` + return status === 403 || status === 404 + ? new ConnectorListingScopeUnavailableError(described, status) + : new Error(described) +} + +/** + * `syncContext` entry the members-mode crawl sets on every listing it runs + * under one member's own token. A connector that walks several scopes reads + * it to tell that a scope the caller cannot reach is simply absent from that + * member's complete listing, where the same failure under a shared credential + * is a cap or an error. + */ +export const PER_MEMBER_LISTING_CONTEXT = { perMemberListing: true } as const + +export function isPerMemberListing(syncContext: Record | undefined): boolean { + return syncContext?.perMemberListing === true +} + +/** + * Whether a folder request that failed while walking a Microsoft Graph drive + * can be left out of the listing: under a member's own token a descendant + * folder Graph reports as unreachable (403, 404) is simply not shared with + * them, so their listing stays complete without it and their access to its + * files is withdrawn. The configured root is the whole scope, which the + * members-mode crawl reads as a complete listing of nothing, and a shared + * credential never skips: dropping the folder's files would read as deletions. + */ +export function isSkippableMicrosoftGraphFolderError( + error: unknown, + syncContext: Record | undefined, + isRootFolder: boolean +): boolean { + return !isRootFolder && isListingScopeUnavailableError(error) && isPerMemberListing(syncContext) +} diff --git a/apps/sim/connectors/zoom/meta.ts b/apps/sim/connectors/zoom/meta.ts index 39be56551e4..b23e4d7627a 100644 --- a/apps/sim/connectors/zoom/meta.ts +++ b/apps/sim/connectors/zoom/meta.ts @@ -19,6 +19,7 @@ export const zoomConnectorMeta: ConnectorMeta = { supportsIncrementalSync: true, + permissionScopedListing: { capFieldIds: ['maxRecordings'] }, configFields: [ { id: 'lookback', diff --git a/apps/sim/ee/workspace-forking/lib/copy/copy-resources.ts b/apps/sim/ee/workspace-forking/lib/copy/copy-resources.ts index 54d01bc3b32..bbb32d2cdff 100644 --- a/apps/sim/ee/workspace-forking/lib/copy/copy-resources.ts +++ b/apps/sim/ee/workspace-forking/lib/copy/copy-resources.ts @@ -47,6 +47,7 @@ import { type DurableSecretProvenance, hashDurableSecretProvenanceValue, } from '@/lib/execution/durable-secret-provenance' +import { WORKSPACE_ACCESS_TOKEN } from '@/lib/knowledge/access/types' import { createKnowledgeDocumentSourceValue, type KnowledgeDocumentSourceValue, @@ -886,6 +887,7 @@ async function createForkDocumentPlaceholders(params: { fileSize: 0, deletedAt: null, archivedAt: now, + acl: [WORKSPACE_ACCESS_TOKEN], }) record('knowledge_document', doc.id, childDocId) kbEntry.documentIdMap[doc.id] = childDocId @@ -995,6 +997,7 @@ export async function planForkMappedKbDocumentCopies(params: { fileSize: 0, deletedAt: null, archivedAt: now, + acl: [WORKSPACE_ACCESS_TOKEN], }) } docIdMap.set(doc.id, childDocId) @@ -1597,6 +1600,7 @@ async function ensureKbDocumentPlaceholder( archivedAt: new Date(), deletedAt: null, uploadedBy: userId, + acl: [WORKSPACE_ACCESS_TOKEN], }) .onConflictDoNothing({ target: document.id }) } @@ -1764,6 +1768,8 @@ async function copyKbDocument(params: { deletedAt: null, uploadedBy: userId, secretProvenanceVersion: sourceSecretContext.tracked ? 1 : null, + /** A copy has no connector and therefore no source-derived access; it is a workspace document. */ + acl: [WORKSPACE_ACCESS_TOKEN], } const copiedSource = createKnowledgeDocumentSourceValue(copiedValues) const finalizedStorageKey = await finalizeKbDocument({ diff --git a/apps/sim/executor/execution/block-executor.ts b/apps/sim/executor/execution/block-executor.ts index be84046151a..744f6b63123 100644 --- a/apps/sim/executor/execution/block-executor.ts +++ b/apps/sim/executor/execution/block-executor.ts @@ -331,6 +331,7 @@ export class BlockExecutor { fileKeys: blockCtx.fileKeys, allowLargeValueWorkflowScope: blockCtx.allowLargeValueWorkflowScope, userId: blockCtx.userId, + principal: blockCtx.principal, maxBytes: blockCtx.base64MaxBytes, preserveLargeValueMetadata: true, })) as NormalizedBlockOutput diff --git a/apps/sim/executor/handlers/agent/agent-handler.ts b/apps/sim/executor/handlers/agent/agent-handler.ts index ca3098dde42..857643acec7 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.ts @@ -1730,6 +1730,7 @@ export class AgentBlockHandler implements BlockHandler { fileKeys: ctx.fileKeys, allowLargeValueWorkflowScope: ctx.allowLargeValueWorkflowScope, userId: ctx.userId, + principal: ctx.principal, logger, maxBytes: inlineMaxBytes, onServableFileContributors: async (file, contributors) => { diff --git a/apps/sim/executor/utils/credential-token.test.ts b/apps/sim/executor/utils/credential-token.test.ts index c4a7d64b160..d23fe290c26 100644 --- a/apps/sim/executor/utils/credential-token.test.ts +++ b/apps/sim/executor/utils/credential-token.test.ts @@ -4,12 +4,19 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import type { ExecutorDelegationOrigin } from '@/executor/types' -const { mockBindExecutorManagedOAuthDelegation, mockResolveCredentialAccessToken } = vi.hoisted( - () => ({ - mockBindExecutorManagedOAuthDelegation: vi.fn(), - mockResolveCredentialAccessToken: vi.fn(), - }) -) +const { + mockBindExecutorManagedOAuthDelegation, + mockCreateCopilotManagedOAuthPrincipal, + mockResolveCredentialAccessToken, +} = vi.hoisted(() => ({ + mockBindExecutorManagedOAuthDelegation: vi.fn(), + mockCreateCopilotManagedOAuthPrincipal: vi.fn(), + mockResolveCredentialAccessToken: vi.fn(), +})) + +vi.mock('@/lib/credentials/application/copilot-managed-oauth-delegation', () => ({ + createCopilotManagedOAuthPrincipal: mockCreateCopilotManagedOAuthPrincipal, +})) vi.mock('@/lib/oauth/token-resolution', () => ({ resolveCredentialAccessToken: mockResolveCredentialAccessToken, @@ -94,6 +101,51 @@ describe('resolveExecutorCredentialToken', () => { expect(mockBindExecutorManagedOAuthDelegation).toHaveBeenCalledWith(ORIGIN, 'managed-1') }) + it('proves a Chat turn through the copilot principal when there is no workflow run', async () => { + mockCreateCopilotManagedOAuthPrincipal.mockReturnValue({ kind: 'delegated' }) + const copilotExecutionContext = { + userId: 'user-1', + workspaceId: 'ws-1', + chatId: 'chat-1', + toolCallId: 'call-1', + copilotToolExecution: true as const, + } + + await resolveExecutorCredentialToken({ + requestId: 'req-1', + credentialId: 'cred-1', + userId: 'user-1', + copilotExecutionContext, + }) + + const input = mockResolveCredentialAccessToken.mock.calls[0][0] + await input.resolveManagedPrincipal('managed-1') + expect(mockCreateCopilotManagedOAuthPrincipal).toHaveBeenCalledWith( + copilotExecutionContext, + 'managed-1' + ) + expect(mockBindExecutorManagedOAuthDelegation).not.toHaveBeenCalled() + }) + + it('leaves managed credentials unproven for a context that is not a trusted Chat call', async () => { + for (const copilotExecutionContext of [ + { userId: 'user-1', workspaceId: 'ws-1' }, + { userId: 'user-1', workspaceId: 'ws-1', copilotToolExecution: true as const }, + ]) { + mockResolveCredentialAccessToken.mockClear() + await resolveExecutorCredentialToken({ + requestId: 'req-1', + credentialId: 'cred-1', + userId: 'user-1', + copilotExecutionContext, + }) + + expect( + mockResolveCredentialAccessToken.mock.calls[0][0].resolveManagedPrincipal + ).toBeUndefined() + } + }) + it('fails before dispatch when the origin lacks current workflow authority', async () => { await expect( resolveExecutorCredentialToken({ diff --git a/apps/sim/executor/utils/credential-token.ts b/apps/sim/executor/utils/credential-token.ts index 6f02767477d..f374f58c79f 100644 --- a/apps/sim/executor/utils/credential-token.ts +++ b/apps/sim/executor/utils/credential-token.ts @@ -1,5 +1,7 @@ import { createLogger } from '@sim/logger' import { AuthType } from '@/lib/auth/hybrid' +import type { CopilotExecutionContext } from '@/lib/copilot/auth/application-delegation' +import { createCopilotManagedOAuthPrincipal } from '@/lib/credentials/application/copilot-managed-oauth-delegation' import { bindExecutorManagedOAuthDelegation } from '@/lib/credentials/application/managed-oauth-delegation' import { type CredentialTokenPayload, @@ -24,6 +26,11 @@ export interface ResolveExecutorCredentialTokenParams { enforceCredentialAccess?: boolean /** Proves managed-credential delegations in-process when the run carries one. */ executorDelegationOrigin?: ExecutorDelegationOrigin + /** + * The trusted Chat tool call this token is for, when there is no workflow + * run: it proves the signed-in user's own Credential Group credential. + */ + copilotExecutionContext?: CopilotExecutionContext } /** @@ -36,12 +43,33 @@ export interface ResolveExecutorCredentialTokenParams { export async function resolveExecutorCredentialToken( params: ResolveExecutorCredentialTokenParams ): Promise { - const { requestId, credentialId, userId, workflowId, toolId, executorDelegationOrigin } = params + const { + requestId, + credentialId, + userId, + workflowId, + toolId, + executorDelegationOrigin, + copilotExecutionContext, + } = params if (executorDelegationOrigin && !executorDelegationOrigin.currentWorkflow) { throw new Error('Managed credential delegation is missing current workflow authority') } + /** + * A Chat proof needs the per-call id the delegation is minted under; a + * context that lacks it is not a Chat tool call and leaves managed + * credentials unproven, so the resolver answers with its own refusal. + */ + const resolveManagedPrincipal = executorDelegationOrigin + ? (managedCredentialId: string) => + bindExecutorManagedOAuthDelegation(executorDelegationOrigin, managedCredentialId) + : copilotExecutionContext?.copilotToolExecution && copilotExecutionContext.toolCallId + ? async (managedCredentialId: string) => + createCopilotManagedOAuthPrincipal(copilotExecutionContext, managedCredentialId) + : undefined + const result = await resolveCredentialAccessToken({ requestId, credentialId, @@ -55,10 +83,7 @@ export async function resolveExecutorCredentialToken( userId, authType: AuthType.INTERNAL_JWT, }), - resolveManagedPrincipal: executorDelegationOrigin - ? (managedCredentialId: string) => - bindExecutorManagedOAuthDelegation(executorDelegationOrigin, managedCredentialId) - : undefined, + resolveManagedPrincipal, }) if (!result.ok) { diff --git a/apps/sim/executor/variables/resolvers/reference-async.server.ts b/apps/sim/executor/variables/resolvers/reference-async.server.ts index 8f40d04b058..e4080d1c768 100644 --- a/apps/sim/executor/variables/resolvers/reference-async.server.ts +++ b/apps/sim/executor/variables/resolvers/reference-async.server.ts @@ -80,6 +80,7 @@ async function hydrateExplicitBase64( fileKeys: context.executionContext.fileKeys, allowLargeValueWorkflowScope: context.executionContext.allowLargeValueWorkflowScope, userId: context.executionContext.userId, + principal: context.executionContext.principal, maxBytes: context.executionContext.base64MaxBytes, }) if (!hydrated.base64) { diff --git a/apps/sim/executor/variables/resolvers/reference.ts b/apps/sim/executor/variables/resolvers/reference.ts index a5005fcf4bc..e2db9898eb9 100644 --- a/apps/sim/executor/variables/resolvers/reference.ts +++ b/apps/sim/executor/variables/resolvers/reference.ts @@ -1,3 +1,4 @@ +import type { WorkflowExecutionPrincipal } from '@sim/auth/principal' import { materializeLargeValueRefSync, materializeLargeValueRefSyncOrThrow, @@ -19,6 +20,8 @@ export interface PathNavigationExecutionContext { fileKeys?: string[] allowLargeValueWorkflowScope?: boolean userId?: string + /** The principal behind the run; knowledge-base files hydrated along the path are read as them. */ + principal?: WorkflowExecutionPrincipal metadata?: { requestId?: string } base64MaxBytes?: number } diff --git a/apps/sim/hooks/queries/kb/connectors.test.ts b/apps/sim/hooks/queries/kb/connectors.test.ts index 2a2dde7dee4..8fb809cea98 100644 --- a/apps/sim/hooks/queries/kb/connectors.test.ts +++ b/apps/sim/hooks/queries/kb/connectors.test.ts @@ -12,6 +12,7 @@ const mocks = vi.hoisted(() => ({ cancelQueries: vi.fn(), getQueryData: vi.fn(), setQueryData: vi.fn(), + setQueriesData: vi.fn(), invalidateQueries: vi.fn(), })) @@ -24,6 +25,7 @@ vi.mock('@tanstack/react-query', () => ({ cancelQueries: mocks.cancelQueries, getQueryData: mocks.getQueryData, setQueryData: mocks.setQueryData, + setQueriesData: mocks.setQueriesData, invalidateQueries: mocks.invalidateQueries, })), })) @@ -41,14 +43,31 @@ import { CONNECTOR_SYNC_POLL_INTERVAL_MS, connectorKeys, isConnectorSyncingOrPending, + memberConnectorKeys, useConnectorDetail, useConnectorDocuments, useConnectorList, useTriggerSync, + type WorkspaceMemberConnector, } from '@/hooks/queries/kb/connectors' const KB_ID = 'kb-1' +function makeMemberConnector( + overrides: Partial = {} +): WorkspaceMemberConnector { + return { + knowledgeBaseId: KB_ID, + knowledgeBaseName: 'Sim Search', + connectorId: 'connector-1', + connectorType: 'hubspot', + memberSyncStatus: 'idle', + viewerMembership: 'connected', + viewerDocumentCount: 0, + ...overrides, + } +} + function makeConnector(overrides: Partial = {}): ConnectorData { return { id: 'connector-1', @@ -250,6 +269,63 @@ describe('useTriggerSync optimistic state', () => { expect(rolledBack?.find((c) => c.id === 'connector-1')?.status).toBe('active') expect(rolledBack?.find((c) => c.id === 'connector-2')?.status).toBe('pending') }) + + /** + * The Search surface reads the member sync status from the workspace + * member-connector list, which has no poll of its own, so a members-mode + * trigger patches that cache too and a refused trigger refetches it. + */ + it('queues a members connector in the workspace member-connector list as well', async () => { + const existing = [ + makeConnector({ id: 'connector-1', accessMode: 'members', memberSyncStatus: 'idle' }), + ] + mocks.getQueryData.mockReturnValue(existing) + + useTriggerSync() + const options = capturedMutationOptions() + const context = await options.onMutate({ knowledgeBaseId: KB_ID, connectorId: 'connector-1' }) + + expect(mocks.setQueriesData).toHaveBeenCalledWith( + { queryKey: memberConnectorKeys.lists() }, + expect.any(Function) + ) + const patchMemberList = mocks.setQueriesData.mock.calls.at(-1)?.[1] as ( + connectors: WorkspaceMemberConnector[] | undefined + ) => WorkspaceMemberConnector[] | undefined + const memberList = [ + makeMemberConnector({ connectorId: 'connector-1', memberSyncStatus: 'idle' }), + makeMemberConnector({ connectorId: 'connector-2', memberSyncStatus: 'idle' }), + ] + expect(patchMemberList(memberList)?.map((c) => c.memberSyncStatus)).toEqual(['pending', 'idle']) + expect(patchMemberList(undefined)).toBeUndefined() + + options.onError( + new Error('boom'), + { knowledgeBaseId: KB_ID, connectorId: 'connector-1' }, + context + ) + expect(mocks.invalidateQueries).toHaveBeenCalledWith({ + queryKey: memberConnectorKeys.lists(), + }) + }) + + it('leaves the workspace member-connector list alone for a workspace connector', async () => { + mocks.getQueryData.mockReturnValue([makeConnector({ status: 'active' })]) + + useTriggerSync() + const options = capturedMutationOptions() + const context = await options.onMutate({ knowledgeBaseId: KB_ID, connectorId: 'connector-1' }) + options.onError( + new Error('boom'), + { knowledgeBaseId: KB_ID, connectorId: 'connector-1' }, + context + ) + + expect(mocks.setQueriesData).not.toHaveBeenCalled() + expect(mocks.invalidateQueries).not.toHaveBeenCalledWith({ + queryKey: memberConnectorKeys.lists(), + }) + }) }) interface ConnectorDocumentsPage { diff --git a/apps/sim/hooks/queries/kb/connectors.ts b/apps/sim/hooks/queries/kb/connectors.ts index ba92aff2e9e..2a8611c21ce 100644 --- a/apps/sim/hooks/queries/kb/connectors.ts +++ b/apps/sim/hooks/queries/kb/connectors.ts @@ -11,20 +11,40 @@ import { type ConnectorData, type ConnectorDetailData, type ConnectorDocumentsData, + type ConnectorMemberSummary, + type ConnectSimSearchConnectorBody, + connectSimSearchConnectorContract, createKnowledgeConnectorContract, deleteKnowledgeConnectorContract, getKnowledgeConnectorContract, listKnowledgeConnectorDocumentsContract, listKnowledgeConnectorsContract, + listWorkspaceMemberConnectorsContract, + type MemberSyncLogData, patchKnowledgeConnectorDocumentsContract, + type StartKnowledgeConnectorMemberEnrollmentData, type SyncLogData, + startKnowledgeConnectorMemberEnrollmentContract, triggerKnowledgeConnectorSyncContract, + type UpdateConnectorAccessBody, + updateKnowledgeConnectorAccessContract, updateKnowledgeConnectorContract, + type ViewerConnectorMembership, + type WorkspaceMemberConnector, } from '@/lib/api/contracts/knowledge' import { MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE } from '@/lib/knowledge/constants' import { knowledgeKeys } from '@/hooks/queries/utils/knowledge-keys' -export type { ConnectorData, ConnectorDetailData, SyncLogData } +export type { + ViewerConnectorMembership, + WorkspaceMemberConnector, + ConnectorData, + ConnectorDetailData, + ConnectorMemberSummary, + MemberSyncLogData, + SyncLogData, + UpdateConnectorAccessBody, +} export const CONNECTOR_LIST_STALE_TIME = 30 * 1000 export const CONNECTOR_DETAIL_STALE_TIME = 30 * 1000 @@ -80,8 +100,14 @@ export const CONNECTOR_SYNC_POLL_INTERVAL_MS = 3000 */ export function isConnectorSyncingOrPending(connector: { status: ConnectorData['status'] + memberSyncStatus?: ConnectorData['memberSyncStatus'] }): boolean { - return connector.status === 'pending' || connector.status === 'syncing' + return ( + connector.status === 'pending' || + connector.status === 'syncing' || + connector.memberSyncStatus === 'pending' || + connector.memberSyncStatus === 'running' + ) } export function useConnectorList(knowledgeBaseId?: string) { @@ -128,20 +154,22 @@ export function useConnectorDetail(knowledgeBaseId?: string, connectorId?: strin * never starting that poll, and showing stale sync history behind the list's * spinner. */ +type ConnectorStatusPatch = Pick | Pick + function setCachedConnectorStatus( queryClient: QueryClient, knowledgeBaseId: string, connectorId: string, - status: ConnectorData['status'] + patch: ConnectorStatusPatch ) { queryClient.setQueryData(connectorKeys.lists(knowledgeBaseId), (connectors) => connectors?.map((connector) => - connector.id === connectorId ? { ...connector, status } : connector + connector.id === connectorId ? { ...connector, ...patch } : connector ) ) queryClient.setQueryData( connectorKeys.detail(knowledgeBaseId, connectorId), - (detail) => (detail ? { ...detail, status } : detail) + (detail) => (detail ? { ...detail, ...patch } : detail) ) } @@ -150,9 +178,9 @@ function setCachedConnectorStatus( * which is all `onError` needs to undo it — the mutation variables already * carry the ids. * - * Both status-changing mutations resolve into the same list, so they share this - * write instead of each keeping a local `Set` of in-flight ids alongside it — - * that duplicated the server's own state and could not survive a remount. + * The pause and resume mutations share this write instead of each keeping a + * local `Set` of in-flight ids alongside it — that duplicated the server's own + * state and could not survive a remount. * * Deliberately not a snapshot of the whole array: two connectors can be in * flight at once, and restoring a whole-list snapshot would roll the other @@ -169,11 +197,47 @@ function optimisticallySetConnectorStatus( .getQueryData(connectorKeys.lists(knowledgeBaseId)) ?.find((connector) => connector.id === connectorId)?.status - setCachedConnectorStatus(queryClient, knowledgeBaseId, connectorId, status) + setCachedConnectorStatus(queryClient, knowledgeBaseId, connectorId, { status }) return previousStatus } +/** + * The optimistic "queued" write for a sync trigger, on whichever engine the + * connector runs: a members connector queues a member run, so its content + * status must not flip. The Search surface reads the same member sync status + * from the workspace member-connector list, so that cache is queued too; it + * has no poll to reconcile it, and the write cannot stop one, so it is patched + * rather than refetched. Returns what to restore if the trigger is refused. + */ +function optimisticallyQueueSync( + queryClient: QueryClient, + knowledgeBaseId: string, + connectorId: string +): ConnectorStatusPatch | undefined { + const cached = queryClient + .getQueryData(connectorKeys.lists(knowledgeBaseId)) + ?.find((connector) => connector.id === connectorId) + if (!cached) return undefined + if (cached.accessMode === 'members') { + setCachedConnectorStatus(queryClient, knowledgeBaseId, connectorId, { + memberSyncStatus: 'pending', + }) + queryClient.setQueriesData( + { queryKey: memberConnectorKeys.lists() }, + (connectors) => + connectors?.map((connector) => + connector.connectorId === connectorId + ? { ...connector, memberSyncStatus: 'pending' } + : connector + ) + ) + return { memberSyncStatus: cached.memberSyncStatus } + } + setCachedConnectorStatus(queryClient, knowledgeBaseId, connectorId, { status: 'pending' }) + return { status: cached.status } +} + interface CreateConnectorParams { knowledgeBaseId: string connectorType: string @@ -181,6 +245,9 @@ interface CreateConnectorParams { apiKey?: string sourceConfig: Record syncIntervalMinutes?: number + accessMode?: 'workspace' | 'members' + credentialGroupId?: string + credentialGroupOptionId?: string } async function createConnector({ @@ -207,6 +274,8 @@ export function useCreateConnector() { */ onSettled: (_data, _error, { knowledgeBaseId }) => { queryClient.invalidateQueries({ queryKey: connectorKeys.all(knowledgeBaseId) }) + queryClient.invalidateQueries({ queryKey: knowledgeKeys.lists() }) + queryClient.invalidateQueries({ queryKey: memberConnectorKeys.lists() }) }, }) } @@ -251,7 +320,9 @@ export function useUpdateConnector() { }, onError: (_error, { knowledgeBaseId, connectorId }, previousStatus) => { if (previousStatus) { - setCachedConnectorStatus(queryClient, knowledgeBaseId, connectorId, previousStatus) + setCachedConnectorStatus(queryClient, knowledgeBaseId, connectorId, { + status: previousStatus, + }) } }, onSettled: (_data, _error, { knowledgeBaseId }) => { @@ -260,6 +331,115 @@ export function useUpdateConnector() { }) } +interface UpdateConnectorAccessParams { + knowledgeBaseId: string + connectorId: string + access: UpdateConnectorAccessBody +} + +async function updateConnectorAccess({ + knowledgeBaseId, + connectorId, + access, +}: UpdateConnectorAccessParams): Promise { + const result = await requestJson(updateKnowledgeConnectorAccessContract, { + params: { id: knowledgeBaseId, connectorId }, + body: access, + }) + + return result.data +} + +interface StartConnectorMemberEnrollmentParams { + knowledgeBaseId: string + connectorId: string +} + +async function startConnectorMemberEnrollment({ + knowledgeBaseId, + connectorId, +}: StartConnectorMemberEnrollmentParams): Promise { + const response = await requestJson(startKnowledgeConnectorMemberEnrollmentContract, { + params: { id: knowledgeBaseId, connectorId }, + }) + return response.data +} + +export const memberConnectorKeys = { + all: ['member-connectors'] as const, + lists: () => [...memberConnectorKeys.all, 'list'] as const, + list: (workspaceId?: string) => [...memberConnectorKeys.lists(), workspaceId ?? ''] as const, +} + +export const WORKSPACE_MEMBER_CONNECTORS_STALE_TIME = 30 * 1000 +/** While a connected source is still indexing for the viewer, its state is worth asking for again. */ +const WORKSPACE_MEMBER_CONNECTORS_INDEXING_POLL_MS = 5 * 1000 + +async function fetchWorkspaceMemberConnectors( + workspaceId: string, + signal?: AbortSignal +): Promise { + const response = await requestJson(listWorkspaceMemberConnectorsContract, { + query: { workspaceId }, + signal, + }) + return response.data +} + +/** Every per-member connector in the workspace and where the viewer stands with each. */ +export function useWorkspaceMemberConnectors( + workspaceId?: string, + options?: { enabled?: boolean } +) { + return useQuery({ + queryKey: memberConnectorKeys.list(workspaceId), + queryFn: ({ signal }) => fetchWorkspaceMemberConnectors(workspaceId as string, signal), + enabled: Boolean(workspaceId) && (options?.enabled ?? true), + staleTime: WORKSPACE_MEMBER_CONNECTORS_STALE_TIME, + refetchInterval: (query) => + query.state.data?.some( + (connector) => + connector.viewerMembership === 'connected' && + (connector.memberSyncStatus === 'pending' || connector.memberSyncStatus === 'running') + ) + ? WORKSPACE_MEMBER_CONNECTORS_INDEXING_POLL_MS + : false, + placeholderData: keepPreviousData, + }) +} + +/** Mints the viewer's enrollment link for a per-member connector; the caller navigates to it. */ +export function useStartConnectorMemberEnrollment() { + return useMutation({ mutationFn: startConnectorMemberEnrollment }) +} + +/** + * Moves a connector between workspace and members mode. The switch rewrites + * document access, so everything under the base is refetched: the connector + * list and detail for the new mode and member state, and the document lists + * and per-document caches whose rows and chunks may have become hidden or + * visible. + */ +export function useUpdateConnectorAccess() { + const queryClient = useQueryClient() + + return useMutation({ + mutationFn: updateConnectorAccess, + onSettled: (_data, _error, { knowledgeBaseId }) => { + queryClient.invalidateQueries({ queryKey: connectorKeys.all(knowledgeBaseId) }) + queryClient.invalidateQueries({ queryKey: knowledgeKeys.documentLists(knowledgeBaseId) }) + queryClient.invalidateQueries({ queryKey: knowledgeKeys.documentDetails(knowledgeBaseId) }) + queryClient.invalidateQueries({ + queryKey: knowledgeKeys.detail(knowledgeBaseId), + exact: true, + }) + /** The base list says whether any connector syncs per member, and the Search tab lists them. */ + queryClient.invalidateQueries({ queryKey: knowledgeKeys.lists() }) + queryClient.invalidateQueries({ queryKey: memberConnectorKeys.lists() }) + }, + }) +} + interface DeleteConnectorParams { knowledgeBaseId: string connectorId: string @@ -290,6 +470,7 @@ export function useDeleteConnector() { */ onSettled: (_data, _error, { knowledgeBaseId, deleteDocuments }) => { queryClient.invalidateQueries({ queryKey: connectorKeys.all(knowledgeBaseId) }) + queryClient.invalidateQueries({ queryKey: memberConnectorKeys.lists() }) queryClient.invalidateQueries({ queryKey: knowledgeKeys.documentLists(knowledgeBaseId) }) queryClient.invalidateQueries({ queryKey: knowledgeKeys.detail(knowledgeBaseId), @@ -304,6 +485,7 @@ export function useDeleteConnector() { if (deleteDocuments) { queryClient.invalidateQueries({ queryKey: knowledgeKeys.documentDetails(knowledgeBaseId) }) } + queryClient.invalidateQueries({ queryKey: knowledgeKeys.lists() }) }, }) } @@ -338,18 +520,30 @@ export function useTriggerSync() { * takes over through `pending` → `syncing` → `active`. */ onMutate: async ({ knowledgeBaseId, connectorId }) => { - await queryClient.cancelQueries({ queryKey: connectorKeys.all(knowledgeBaseId) }) - return optimisticallySetConnectorStatus(queryClient, knowledgeBaseId, connectorId, 'pending') + await Promise.all([ + queryClient.cancelQueries({ queryKey: connectorKeys.all(knowledgeBaseId) }), + queryClient.cancelQueries({ queryKey: memberConnectorKeys.lists() }), + ]) + return optimisticallyQueueSync(queryClient, knowledgeBaseId, connectorId) }, /** * Rolling back also stops the poll the optimistic `pending` started, so a * refused sync does not leave the row spinning. */ - onError: (_error, { knowledgeBaseId, connectorId }, previousStatus) => { - if (previousStatus) { - setCachedConnectorStatus(queryClient, knowledgeBaseId, connectorId, previousStatus) + onError: (_error, { knowledgeBaseId, connectorId }, previous) => { + if (previous) { + setCachedConnectorStatus(queryClient, knowledgeBaseId, connectorId, previous) + } + /** + * The member-connector list took the same optimistic `pending`; a refetch + * is its rollback, and the connector list's own status was restored above, + * so it is not refetched over concurrent optimistic patches. + */ + if (previous && 'memberSyncStatus' in previous) { + queryClient.invalidateQueries({ queryKey: memberConnectorKeys.lists() }) + } else { + queryClient.invalidateQueries({ queryKey: connectorKeys.all(knowledgeBaseId) }) } - queryClient.invalidateQueries({ queryKey: connectorKeys.all(knowledgeBaseId) }) }, /** * Deliberately no invalidation on success. The route answers without @@ -504,3 +698,28 @@ export function useRestoreConnectorDocument() { invalidateConnectorDocumentChange(queryClient, variables), }) } + +async function connectSimSearchConnector(body: ConnectSimSearchConnectorBody) { + const result = await requestJson(connectSimSearchConnectorContract, { body }) + return result.data +} + +/** + * One click on a Sim Search source: the source's per-member connector exists + * afterwards and the viewer has their enrollment link. The member list and the + * base list both gain a row on a first connect. + */ +export function useConnectSimSearchConnector() { + const queryClient = useQueryClient() + return useMutation({ + mutationFn: connectSimSearchConnector, + onSuccess: (data) => { + /** A first connect added a connector to the base; its own list is open on the settings page. */ + queryClient.invalidateQueries({ queryKey: connectorKeys.all(data.knowledgeBaseId) }) + }, + onSettled: () => { + queryClient.invalidateQueries({ queryKey: memberConnectorKeys.lists() }) + queryClient.invalidateQueries({ queryKey: knowledgeKeys.lists() }) + }, + }) +} diff --git a/apps/sim/hooks/queries/kb/knowledge.ts b/apps/sim/hooks/queries/kb/knowledge.ts index 4c064e479c9..fe885869cfa 100644 --- a/apps/sim/hooks/queries/kb/knowledge.ts +++ b/apps/sim/hooks/queries/kb/knowledge.ts @@ -41,6 +41,7 @@ import { restoreKnowledgeBaseContract, type SaveDocumentTagDefinitionsResult, saveDocumentTagDefinitionsContract, + searchWorkspaceKnowledgeContract, type TagDefinitionData, type TagUsageData, type UpdateKnowledgeDocumentResponseData, @@ -48,6 +49,8 @@ import { updateKnowledgeChunkContract, updateKnowledgeDocumentContract, updateKnowledgeDocumentTagsContract, + type WorkspaceKnowledgeSearchBody, + type WorkspaceKnowledgeSearchResult, } from '@/lib/api/contracts/knowledge' import type { ChunkingStrategy, StrategyOptions } from '@/lib/chunkers/types' import type { DocumentSortField, SortOrder } from '@/lib/knowledge/documents/types' @@ -74,6 +77,7 @@ export const KNOWLEDGE_DOCUMENT_DETAIL_STALE_TIME = 60 * 1000 export const KNOWLEDGE_DOCUMENT_LIST_STALE_TIME = 60 * 1000 export const KNOWLEDGE_CHUNK_LIST_STALE_TIME = 60 * 1000 export const KNOWLEDGE_CHUNK_SEARCH_STALE_TIME = 60 * 1000 +export const WORKSPACE_KNOWLEDGE_SEARCH_STALE_TIME = 60 * 1000 export const KNOWLEDGE_TAG_DEFINITION_LIST_STALE_TIME = 60 * 1000 export const KNOWLEDGE_TAG_USAGE_STALE_TIME = 60 * 1000 export const KNOWLEDGE_DOCUMENT_TAG_DEFINITION_LIST_STALE_TIME = 60 * 1000 @@ -1153,3 +1157,38 @@ export function useBulkDeleteKnowledgeBases(workspaceId: string) { }, }) } + +async function searchWorkspaceKnowledge( + body: WorkspaceKnowledgeSearchBody, + signal?: AbortSignal +): Promise { + const data = await requestJson(searchWorkspaceKnowledgeContract, { body, signal }) + return data.data.results +} + +/** + * What the signed-in person may read that matches `query`, across the given + * knowledge bases. Off until there is a query and a base to search. + */ +export function useWorkspaceKnowledgeSearch( + workspaceId: string | undefined, + knowledgeBaseIds: readonly string[], + query: string +) { + const trimmed = query.trim() + return useQuery({ + queryKey: knowledgeKeys.search(workspaceId, knowledgeBaseIds, trimmed), + queryFn: ({ signal }) => + searchWorkspaceKnowledge( + { + workspaceId: workspaceId as string, + knowledgeBaseIds: [...knowledgeBaseIds], + query: trimmed, + }, + signal + ), + enabled: Boolean(workspaceId) && knowledgeBaseIds.length > 0 && trimmed.length > 0, + staleTime: WORKSPACE_KNOWLEDGE_SEARCH_STALE_TIME, + placeholderData: keepPreviousData, + }) +} diff --git a/apps/sim/hooks/queries/utils/knowledge-keys.ts b/apps/sim/hooks/queries/utils/knowledge-keys.ts index e47704214d7..b4d2611d14a 100644 --- a/apps/sim/hooks/queries/utils/knowledge-keys.ts +++ b/apps/sim/hooks/queries/utils/knowledge-keys.ts @@ -30,6 +30,14 @@ export const knowledgeKeys = { details: () => [...knowledgeKeys.all, 'detail'] as const, detail: (knowledgeBaseId?: string) => [...knowledgeKeys.details(), knowledgeBaseId ?? ''] as const, + searches: () => [...knowledgeKeys.all, 'search'] as const, + search: (workspaceId: string | undefined, knowledgeBaseIds: readonly string[], query: string) => + [ + ...knowledgeKeys.searches(), + workspaceId ?? '', + [...knowledgeBaseIds].sort().join(','), + query, + ] as const, tagDefinitions: (knowledgeBaseId: string) => [...knowledgeKeys.detail(knowledgeBaseId), 'tagDefinitions'] as const, tagUsage: (knowledgeBaseId: string) => diff --git a/apps/sim/hooks/use-member-enrollment.test.tsx b/apps/sim/hooks/use-member-enrollment.test.tsx new file mode 100644 index 00000000000..58773f49259 --- /dev/null +++ b/apps/sim/hooks/use-member-enrollment.test.tsx @@ -0,0 +1,119 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + enrollmentMutate: vi.fn(), + sourceConnectionMutate: vi.fn(), + invalidateQueries: vi.fn(), +})) + +vi.mock('@tanstack/react-query', () => ({ + useQueryClient: () => ({ invalidateQueries: mocks.invalidateQueries }), +})) +vi.mock('@/hooks/queries/kb/connectors', () => ({ + memberConnectorKeys: { lists: () => ['member-connectors', 'list'] }, + useStartConnectorMemberEnrollment: () => ({ + mutate: mocks.enrollmentMutate, + submittedAt: 0, + isPending: false, + error: null, + }), + useConnectSimSearchConnector: () => ({ + mutate: mocks.sourceConnectionMutate, + submittedAt: 0, + isPending: false, + error: null, + }), +})) + +import { useMemberEnrollment } from '@/hooks/use-member-enrollment' + +type Enrollment = ReturnType + +let latest: Enrollment | null = null +let root: Root | null = null +let container: HTMLDivElement | null = null + +function Harness({ connected }: { connected: ReadonlySet }) { + latest = useMemberEnrollment({ membershipQueryKeys: [], connectedConnectorIds: connected }) + return null +} + +function mount(connected: ReadonlySet = new Set()) { + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) + act(() => root?.render()) +} + +function enrollment(): Enrollment { + if (!latest) throw new Error('Hook did not render') + return latest +} + +beforeEach(() => { + vi.clearAllMocks() + vi.spyOn(window, 'open').mockReturnValue({ + location: { href: '' }, + close: vi.fn(), + } as unknown as Window) +}) + +afterEach(() => { + if (root) act(() => root?.unmount()) + container?.remove() + root = null + container = null + latest = null + vi.restoreAllMocks() +}) + +describe('useMemberEnrollment', () => { + /** + * The connect that creates a Sim Search source's connector returns its id, + * but the membership list has no row for it until it refetches, so the + * source is awaited by type until then and by id once the row exists. + */ + it('awaits a first-connected source by type until its membership row appears', () => { + mount() + act(() => enrollment().connectSource('workspace-1', 'google_drive')) + + const [, handlers] = mocks.sourceConnectionMutate.mock.calls[0] + act(() => + handlers.onSuccess({ url: 'https://example.test/enroll', connectorId: 'connector-1' }) + ) + + expect(enrollment().isAwaitingSource('google_drive')).toBe(true) + expect(enrollment().isAwaitingSource('slack')).toBe(false) + expect(enrollment().isAwaiting('connector-1')).toBe(true) + }) + + it('stops awaiting a source once the viewer is connected to its connector', () => { + mount() + act(() => enrollment().connectSource('workspace-1', 'google_drive')) + const [, handlers] = mocks.sourceConnectionMutate.mock.calls[0] + act(() => + handlers.onSuccess({ url: 'https://example.test/enroll', connectorId: 'connector-1' }) + ) + + act(() => root?.render()) + + expect(enrollment().isAwaitingSource('google_drive')).toBe(false) + expect(enrollment().isAwaiting('connector-1')).toBe(false) + }) + + it('does not report an enrollment in an existing connector as an awaited source', () => { + mount() + act(() => enrollment().connect('kb-1', 'connector-1')) + const [, handlers] = mocks.enrollmentMutate.mock.calls[0] + act(() => handlers.onSuccess({ url: 'https://example.test/enroll' })) + + expect(enrollment().isAwaiting('connector-1')).toBe(true) + expect(enrollment().isAwaitingSource('google_drive')).toBe(false) + }) +}) diff --git a/apps/sim/hooks/use-member-enrollment.ts b/apps/sim/hooks/use-member-enrollment.ts new file mode 100644 index 00000000000..e1a6ee1c590 --- /dev/null +++ b/apps/sim/hooks/use-member-enrollment.ts @@ -0,0 +1,273 @@ +'use client' + +import { useEffect, useRef, useState } from 'react' +import { createLogger } from '@sim/logger' +import { type QueryKey, useQueryClient } from '@tanstack/react-query' +import type { MemberSyncStatus } from '@/lib/knowledge/types' +import type { SearchConnector } from '@/lib/sim-search/connectors' +import { + memberConnectorKeys, + useConnectSimSearchConnector, + useStartConnectorMemberEnrollment, + type ViewerConnectorMembership, + type WorkspaceMemberConnector, +} from '@/hooks/queries/kb/connectors' + +const logger = createLogger('MemberEnrollment') + +/** How often the membership queries are refreshed while a member connects in another tab. */ +const AWAITING_CONNECTION_POLL_MS = 4_000 +/** How long a connection is awaited before the surface stops refreshing on its own. */ +const AWAITING_CONNECTION_TIMEOUT_MS = 10 * 60_000 +const POPUP_BLOCKED_MESSAGE = 'Allow pop-ups for this site to connect your account.' + +/** Memberships the viewer can act on themselves. */ +export const CONNECTABLE_MEMBERSHIPS: ReadonlySet = new Set([ + 'needs_reauth', + 'invited', + 'not_enrolled', +]) + +/** The label of the one action a connectable membership offers. */ +export function enrollmentActionLabel( + membership: ViewerConnectorMembership, + waiting: boolean +): string { + if (waiting) return 'Open again' + return membership === 'needs_reauth' ? 'Reconnect' : 'Connect' +} + +interface DescribeMembershipInput { + membership: ViewerConnectorMembership + memberSyncStatus: MemberSyncStatus + /** Whether this surface opened an enrollment tab that has not connected yet. */ + waiting: boolean + /** The connector's display name. */ + name: string +} + +/** + * One sentence on where the viewer stands with a per-member connector, shared + * by every surface that shows it so the wording cannot drift between them. + * Null once the viewer is connected and nothing is happening for them. + */ +export function describeMembership({ + membership, + memberSyncStatus, + waiting, + name, +}: DescribeMembershipInput): string | null { + switch (membership) { + case 'connected': + switch (memberSyncStatus) { + case 'pending': + case 'running': + return `Syncing the ${name} documents shared with you. They appear when the sync completes.` + case 'error': + return `The last ${name} sync failed; the documents you already have stay visible while it retries.` + case 'disabled': + return `Syncing ${name} per member is turned off. Ask a workspace admin to turn it back on.` + default: + return null + } + case 'needs_reauth': + return `Reconnect your ${name} account to keep seeing the documents shared with you.` + case 'unverified_email': + return `Verify your email address to see the ${name} documents shared with you.` + case 'revoked': + return `A workspace admin removed your access to ${name} documents.` + default: + return waiting + ? `Finish connecting your ${name} account in the other tab.` + : `Connect your ${name} account to see the documents shared with you.` + } +} + +/** An enrollment tab this surface opened that has not connected yet. */ +interface AwaitingEnrollment { + since: number + /** + * The Sim Search source whose connect created the connector, so the source + * can be told it is awaited before its membership row exists to look it up by. + */ + connectorType: string | null +} + +interface UseMemberEnrollmentProps { + /** Queries this surface reads memberships from, refreshed while a connection is awaited. */ + membershipQueryKeys: readonly QueryKey[] + /** Connector ids the viewer is now connected to; awaiting stops for them. */ + connectedConnectorIds: ReadonlySet +} + +/** + * Lets the viewer connect their own account to a per-member connector, by + * connector or by Sim Search source. Enrollment opens in a new tab, and the + * membership queries are polled meanwhile so the surface that started it + * updates on its own once the account is connected; the workspace-wide + * membership list is refreshed too, so the other surface catches up as well. + * + * The tab is opened in the click itself, before the enrollment link is + * minted, because a tab opened after a network round trip is outside the + * click's activation window and popup blockers swallow it. + */ +export function useMemberEnrollment({ + membershipQueryKeys, + connectedConnectorIds, +}: UseMemberEnrollmentProps) { + const connectedRef = useRef(connectedConnectorIds) + const queryClient = useQueryClient() + const enrollment = useStartConnectorMemberEnrollment() + const sourceConnection = useConnectSimSearchConnector() + const [awaitingSince, setAwaitingSince] = useState>( + () => new Map() + ) + const [popupBlocked, setPopupBlocked] = useState(false) + + useEffect(() => { + connectedRef.current = connectedConnectorIds + }, [connectedConnectorIds]) + + /** + * Polls while any connection is awaited, and once more after the last one + * connects: that tick drops the connected ids, so a token that later needs + * reauthorization is not mistaken for a connection still being awaited. + */ + const awaiting = awaitingSince.size > 0 + useEffect(() => { + if (!awaiting) return + const timer = setInterval(() => { + const now = Date.now() + setAwaitingSince((current) => { + const next = new Map( + [...current].filter( + ([id, { since }]) => + !connectedRef.current.has(id) && now - since < AWAITING_CONNECTION_TIMEOUT_MS + ) + ) + return next.size === current.size ? current : next + }) + for (const queryKey of membershipQueryKeys) { + void queryClient.invalidateQueries({ queryKey }) + } + void queryClient.invalidateQueries({ queryKey: memberConnectorKeys.lists() }) + }, AWAITING_CONNECTION_POLL_MS) + return () => clearInterval(timer) + }, [awaiting, membershipQueryKeys, queryClient]) + + /** Opens the tab inside the click, then sends it wherever `start` mints. */ + const openEnrollment = ( + start: (handlers: { + onSuccess: (url: string, connectorId: string, connectorType?: string) => void + onError: () => void + }) => void + ) => { + const tab = window.open('about:blank', '_blank') + if (!tab) { + setPopupBlocked(true) + return + } + tab.opener = null + setPopupBlocked(false) + start({ + onSuccess: (url, connectorId, connectorType) => { + tab.location.href = url + setAwaitingSince((current) => + new Map(current).set(connectorId, { + since: Date.now(), + connectorType: connectorType ?? null, + }) + ) + }, + onError: () => tab.close(), + }) + } + + const connect = (knowledgeBaseId: string, connectorId: string) => + openEnrollment(({ onSuccess, onError }) => { + enrollment.mutate( + { knowledgeBaseId, connectorId }, + { + onSuccess: ({ url }) => onSuccess(url, connectorId), + onError: (err) => { + onError() + logger.error('Failed to start member enrollment', { error: err.message }) + }, + } + ) + }) + + /** + * Connects a Sim Search source: its per-member connector exists afterwards, + * and the viewer enrolls. The setup fields are read only when this connect + * creates the connector. + */ + const connectSource = ( + workspaceId: string, + connectorType: string, + sourceConfig?: Record + ) => + openEnrollment(({ onSuccess, onError }) => { + sourceConnection.mutate( + { workspaceId, connectorType, sourceConfig }, + { + onSuccess: ({ url, connectorId }) => onSuccess(url, connectorId, connectorType), + onError: (err) => { + onError() + logger.error('Failed to connect a Sim Search source', { error: err.message }) + }, + } + ) + }) + + const [setupConnector, setSetupConnector] = useState(null) + + /** + * One click on a Sim Search source: enroll in its connector when someone + * already connected it, ask for its setup fields when it needs them, and + * otherwise create it and enroll in one step. + */ + const connectSearchSource = ( + workspaceId: string, + connector: SearchConnector, + connection: WorkspaceMemberConnector | undefined + ) => { + if (connection) { + connect(connection.knowledgeBaseId, connection.connectorId) + return + } + if (connector.setupFields.length > 0) { + setSetupConnector(connector) + return + } + connectSource(workspaceId, connector.type) + } + + const isAwaiting = (connectorId: string) => + awaitingSince.has(connectorId) && !connectedConnectorIds.has(connectorId) + + /** + * Whether a Sim Search source is awaited by the connect that created its + * connector: the membership list has no row for it until it refetches, so + * the source cannot be looked up by connector id yet. + */ + const isAwaitingSource = (connectorType: string) => + [...awaitingSince].some( + ([id, awaiting]) => awaiting.connectorType === connectorType && !connectedConnectorIds.has(id) + ) + + /** The surface reports the latest attempt, whichever path made it. */ + const latest = + enrollment.submittedAt >= sourceConnection.submittedAt ? enrollment : sourceConnection + return { + connect, + connectSource, + connectSearchSource, + setupConnector, + closeSetup: () => setSetupConnector(null), + isAwaiting, + isAwaitingSource, + isPending: latest.isPending, + error: popupBlocked ? POPUP_BLOCKED_MESSAGE : (latest.error?.message ?? null), + } +} diff --git a/apps/sim/lib/api/contracts/knowledge/base.ts b/apps/sim/lib/api/contracts/knowledge/base.ts index a0f88bd5f36..5e154df85ec 100644 --- a/apps/sim/lib/api/contracts/knowledge/base.ts +++ b/apps/sim/lib/api/contracts/knowledge/base.ts @@ -204,6 +204,7 @@ export const knowledgeBaseDataSchema = z folderId: z.string().nullable(), docCount: z.number().optional(), connectorTypes: z.array(z.string()).optional(), + hasMemberScopedConnector: z.boolean().optional(), }) .passthrough() export type KnowledgeBaseData = z.output diff --git a/apps/sim/lib/api/contracts/knowledge/connectors.test.ts b/apps/sim/lib/api/contracts/knowledge/connectors.test.ts new file mode 100644 index 00000000000..214523edcec --- /dev/null +++ b/apps/sim/lib/api/contracts/knowledge/connectors.test.ts @@ -0,0 +1,105 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + createConnectorBodySchema, + updateConnectorAccessBodySchema, +} from '@/lib/api/contracts/knowledge/connectors' + +const base = { + connectorType: 'google_drive', + sourceConfig: { folderId: ['f-1'] }, +} + +describe('connector access binding contracts', () => { + it('defaults a create to workspace mode with a credential', () => { + const parsed = createConnectorBodySchema.parse({ ...base, credentialId: 'cred-1' }) + expect(parsed.accessMode).toBe('workspace') + expect(parsed.syncIntervalMinutes).toBe(1440) + }) + + it('lets members mode omit the binding, refuses half a binding, and refuses a credential there', () => { + /** No binding named: the server provisions a credential group for the connector. */ + expect(createConnectorBodySchema.safeParse({ ...base, accessMode: 'members' }).success).toBe( + true + ) + expect( + createConnectorBodySchema.safeParse({ + ...base, + accessMode: 'members', + credentialGroupId: 'group-1', + }).success + ).toBe(false) + expect( + createConnectorBodySchema.safeParse({ + ...base, + accessMode: 'members', + credentialGroupId: 'group-1', + credentialGroupOptionId: 'option-1', + credentialId: 'cred-1', + }).success + ).toBe(false) + expect( + createConnectorBodySchema.safeParse({ + ...base, + accessMode: 'members', + credentialGroupId: 'group-1', + credentialGroupOptionId: 'option-1', + }).success + ).toBe(true) + }) + + it('refuses a group binding on a workspace-mode connector', () => { + expect( + createConnectorBodySchema.safeParse({ + ...base, + credentialId: 'cred-1', + credentialGroupId: 'group-1', + }).success + ).toBe(false) + }) + + it('refuses a mode switch that names no mode', () => { + expect(updateConnectorAccessBodySchema.safeParse({}).success).toBe(false) + expect(updateConnectorAccessBodySchema.safeParse({ credentialId: 'cred-1' }).success).toBe( + false + ) + }) + + it('applies the same rules to a mode switch', () => { + expect( + updateConnectorAccessBodySchema.safeParse({ + accessMode: 'members', + credentialGroupId: 'group-1', + credentialGroupOptionId: 'option-1', + }).success + ).toBe(true) + expect( + updateConnectorAccessBodySchema.safeParse({ + accessMode: 'workspace', + credentialId: 'cred-1', + }).success + ).toBe(true) + expect( + updateConnectorAccessBodySchema.safeParse({ + accessMode: 'workspace', + credentialGroupOptionId: 'option-1', + }).success + ).toBe(false) + expect( + updateConnectorAccessBodySchema.safeParse({ + accessMode: 'members', + credentialGroupId: 'group-1', + credentialGroupOptionId: 'option-1', + credentialId: 'cred-1', + }).success + ).toBe(false) + }) + + it('refuses a switch to workspace mode that names no credential', () => { + const parsed = updateConnectorAccessBodySchema.safeParse({ accessMode: 'workspace' }) + expect(parsed.success).toBe(false) + expect(parsed.error?.issues.map((issue) => issue.path)).toEqual([['credentialId']]) + }) +}) diff --git a/apps/sim/lib/api/contracts/knowledge/connectors.ts b/apps/sim/lib/api/contracts/knowledge/connectors.ts index 5261106be51..1e7ffc1213d 100644 --- a/apps/sim/lib/api/contracts/knowledge/connectors.ts +++ b/apps/sim/lib/api/contracts/knowledge/connectors.ts @@ -4,21 +4,112 @@ import { knowledgeConnectorParamsSchema, successResponseSchema, } from '@/lib/api/contracts/knowledge/shared' -import { booleanQueryFlagSchema } from '@/lib/api/contracts/primitives' +import { booleanQueryFlagSchema, workspaceIdSchema } from '@/lib/api/contracts/primitives' import { defineRouteContract } from '@/lib/api/contracts/types' import { DEFAULT_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE, MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_MUTATION_ITEMS, MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE, } from '@/lib/knowledge/constants' +import { MEMBER_SYNC_STATUSES } from '@/lib/knowledge/types' -export const createConnectorBodySchema = z.object({ - connectorType: z.string().min(1), - credentialId: z.string().min(1).optional(), - apiKey: z.string().min(1).optional(), - sourceConfig: z.record(z.string(), z.unknown()), - syncIntervalMinutes: z.number().int().min(0).default(1440), -}) +/** + * How a connector derives document access. `workspace` syncs as one credential + * and every document is visible to the workspace; `members` crawls once per + * Credential Group member and a document is visible to the members whose crawl + * returned it. `admin` is reserved. + */ +export const connectorAccessModeSchema = z.enum(['workspace', 'members', 'admin']) +export type ConnectorAccessMode = z.output + +/** The modes a caller may put a connector into. */ +export const connectorRequestedAccessModeSchema = z.enum(['workspace', 'members']) + +const connectorAccessBindingShape = { + accessMode: connectorRequestedAccessModeSchema.optional().default('workspace'), + /** Members mode: the Credential Group whose option supplies member credentials. */ + credentialGroupId: z.string().min(1).optional(), + /** Members mode: the option within the group; must collect this connector's provider. */ + credentialGroupOptionId: z.string().min(1).optional(), +} as const + +function requireAccessBinding( + value: { + accessMode: 'workspace' | 'members' + credentialGroupId?: string + credentialGroupOptionId?: string + }, + ctx: z.RefinementCtx +): void { + if (value.accessMode === 'members') { + /** Both name one option, or neither and the server provisions one. */ + if (Boolean(value.credentialGroupId) !== Boolean(value.credentialGroupOptionId)) { + ctx.addIssue({ + code: 'custom', + path: [value.credentialGroupId ? 'credentialGroupOptionId' : 'credentialGroupId'], + message: 'credentialGroupId and credentialGroupOptionId go together', + }) + } + return + } + if (value.credentialGroupId || value.credentialGroupOptionId) { + ctx.addIssue({ + code: 'custom', + path: ['credentialGroupId'], + message: 'A Credential Group binding only applies when accessMode is members', + }) + } +} + +export const createConnectorBodySchema = z + .object({ + connectorType: z.string().min(1), + credentialId: z.string().min(1).optional(), + apiKey: z.string().min(1).optional(), + sourceConfig: z.record(z.string(), z.unknown()), + syncIntervalMinutes: z.number().int().min(0).default(1440), + ...connectorAccessBindingShape, + }) + .superRefine((value, ctx) => { + requireAccessBinding(value, ctx) + if (value.accessMode === 'members' && value.credentialId) { + ctx.addIssue({ + code: 'custom', + path: ['credentialId'], + message: 'A members-mode connector crawls with member credentials, not a credentialId', + }) + } + }) + +/** + * Moves a connector between access modes. Switching to workspace mode needs + * the credential the connector will sync as from then on. + */ +export const updateConnectorAccessBodySchema = z + .object({ + ...connectorAccessBindingShape, + /** A switch names the mode it moves to; nothing is implied by omission. */ + accessMode: connectorRequestedAccessModeSchema, + credentialId: z.string().min(1).optional(), + }) + .superRefine((value, ctx) => { + requireAccessBinding(value, ctx) + if (value.accessMode === 'members' && value.credentialId) { + ctx.addIssue({ + code: 'custom', + path: ['credentialId'], + message: 'A members-mode connector crawls with member credentials, not a credentialId', + }) + } + if (value.accessMode === 'workspace' && !value.credentialId) { + ctx.addIssue({ + code: 'custom', + path: ['credentialId'], + message: 'Switching to workspace mode needs the credentialId the connector syncs as', + }) + } + }) +export type UpdateConnectorAccessBody = z.input export const updateConnectorBodySchema = z.object({ sourceConfig: z.record(z.string(), z.unknown()).optional(), @@ -51,6 +142,17 @@ export const connectorDocumentsPatchBodySchema = z.object({ .max(MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_MUTATION_ITEMS), }) +export const VIEWER_CONNECTOR_MEMBERSHIPS = [ + 'connected', + 'needs_reauth', + 'invited', + 'not_enrolled', + 'revoked', + 'unverified_email', +] as const +export const viewerConnectorMembershipSchema = z.enum(VIEWER_CONNECTOR_MEMBERSHIPS) +export type ViewerConnectorMembership = z.output + export const connectorDataSchema = z .object({ id: z.string(), @@ -67,6 +169,23 @@ export const connectorDataSchema = z lastSyncDocCount: z.number().nullable(), nextSyncAt: z.string().nullable(), consecutiveFailures: z.number(), + accessMode: connectorAccessModeSchema, + /** + * Where the viewer stands with a per-member connector; null for a + * workspace-mode connector, a caller with no person behind it, or where + * per-member access is not available. + */ + viewerMembership: viewerConnectorMembershipSchema.nullable(), + credentialGroupId: z.string().nullable(), + credentialGroupOptionId: z.string().nullable(), + /** Members mode only; `idle` otherwise. */ + memberSyncStatus: z.enum(MEMBER_SYNC_STATUSES), + lastMemberSyncAt: z.string().nullable(), + nextMemberSyncAt: z.string().nullable(), + lastMemberSyncError: z.string().nullable(), + memberSyncConsecutiveFailures: z.number(), + /** A mode switch left its ACL rewrite for the next member run to finish. */ + accessRewritePending: z.boolean(), createdAt: z.string(), updatedAt: z.string(), }) @@ -101,8 +220,46 @@ export const syncLogDataSchema = z .passthrough() export type SyncLogData = z.output +export const memberSyncLogDataSchema = z + .object({ + id: z.string(), + connectorId: z.string(), + status: syncLogStatusSchema, + startedAt: z.string(), + completedAt: z.string().nullable(), + membersClaimed: z.number(), + membersCompleted: z.number(), + membersIncomplete: z.number(), + membersFailed: z.number(), + docsListed: z.number(), + docsAdded: z.number(), + docsUpdated: z.number(), + docsUnchanged: z.number(), + docsHydratedOnce: z.number(), + observationsAdded: z.number(), + observationsRemoved: z.number(), + docsTombstoned: z.number(), + docsResurrected: z.number(), + docsPurged: z.number(), + credentialsAudited: z.number(), + errorMessage: z.string().nullable(), + }) + .passthrough() +export type MemberSyncLogData = z.output + +/** How many of a members-mode connector's members are in each state. */ +export const connectorMemberSummarySchema = z.object({ + active: z.number().int().nonnegative(), + suspended: z.number().int().nonnegative(), + /** Active members whose last complete listing is older than the staleness window. */ + stale: z.number().int().nonnegative(), +}) +export type ConnectorMemberSummary = z.output + export const connectorDetailDataSchema = connectorDataSchema.extend({ syncLogs: z.array(syncLogDataSchema), + memberSyncLogs: z.array(memberSyncLogDataSchema), + members: connectorMemberSummarySchema, }) export type ConnectorDetailData = z.output @@ -170,6 +327,88 @@ export const updateKnowledgeConnectorContract = defineRouteContract({ }, }) +export const updateKnowledgeConnectorAccessContract = defineRouteContract({ + method: 'PATCH', + path: '/api/knowledge/[id]/connectors/[connectorId]/access', + params: knowledgeConnectorParamsSchema, + body: updateConnectorAccessBodySchema, + response: { + mode: 'json', + schema: successResponseSchema(connectorDataSchema), + }, +}) + +export const startKnowledgeConnectorMemberEnrollmentDataSchema = z.object({ + /** The viewer's enrollment link; opening it connects their account. */ + url: z.string().url(), +}) +export type StartKnowledgeConnectorMemberEnrollmentData = z.output< + typeof startKnowledgeConnectorMemberEnrollmentDataSchema +> + +export const startKnowledgeConnectorMemberEnrollmentContract = defineRouteContract({ + method: 'POST', + path: '/api/knowledge/[id]/connectors/[connectorId]/enroll', + params: knowledgeConnectorParamsSchema, + response: { + mode: 'json', + schema: successResponseSchema(startKnowledgeConnectorMemberEnrollmentDataSchema), + }, +}) + +/** A per-member connector as the viewer meets it across the workspace's knowledge bases. */ +export const workspaceMemberConnectorSchema = z.object({ + knowledgeBaseId: z.string(), + knowledgeBaseName: z.string(), + connectorId: z.string(), + connectorType: z.string(), + memberSyncStatus: z.enum(MEMBER_SYNC_STATUSES), + viewerMembership: viewerConnectorMembershipSchema, + /** Documents of this connector the viewer may read right now. */ + viewerDocumentCount: z.number().int().nonnegative(), +}) +export type WorkspaceMemberConnector = z.output + +export const connectSimSearchConnectorBodySchema = z.object({ + workspaceId: workspaceIdSchema, + connectorType: z.string().min(1, 'connectorType cannot be empty').max(100), + /** The source's setup fields, needed only on the connect that creates it. */ + sourceConfig: z.record(z.string(), z.string().max(500)).optional(), +}) +export type ConnectSimSearchConnectorBody = z.input + +/** + * One click on a Sim Search source: the workspace's Sim Search knowledge base + * and per-member connector exist afterwards, and the caller gets the link that + * connects their own account. + */ +export const connectSimSearchConnectorContract = defineRouteContract({ + method: 'POST', + path: '/api/knowledge/sim-search/connect', + body: connectSimSearchConnectorBodySchema, + response: { + mode: 'json', + schema: z.object({ + success: z.literal(true), + data: z.object({ + knowledgeBaseId: z.string(), + connectorId: z.string(), + url: z.string().url(), + }), + }), + }, +}) + +export const listWorkspaceMemberConnectorsContract = defineRouteContract({ + method: 'GET', + path: '/api/knowledge/member-connectors', + query: z.object({ workspaceId: workspaceIdSchema }), + response: { + mode: 'json', + schema: successResponseSchema(z.array(workspaceMemberConnectorSchema)), + }, +}) + export const deleteKnowledgeConnectorContract = defineRouteContract({ method: 'DELETE', path: '/api/knowledge/[id]/connectors/[connectorId]', diff --git a/apps/sim/lib/api/contracts/knowledge/documents.ts b/apps/sim/lib/api/contracts/knowledge/documents.ts index 151cae080e5..9b7508bdbb8 100644 --- a/apps/sim/lib/api/contracts/knowledge/documents.ts +++ b/apps/sim/lib/api/contracts/knowledge/documents.ts @@ -286,7 +286,12 @@ export const documentDataSchema = z connectorType: z.string().nullable().optional(), sourceUrl: z.string().nullable().optional(), }) - .passthrough() + /** + * Strict allow-list. The single-document presenter spreads the whole + * `document` row, so an unlisted column — `storageKey`, and now `acl` — + * would otherwise reach every client that can read a document. + */ + .strip() export type DocumentData = z.output export const documentsPaginationSchema = paginationSchema diff --git a/apps/sim/lib/api/contracts/knowledge/search.ts b/apps/sim/lib/api/contracts/knowledge/search.ts index cf42167554a..d2529f30fb4 100644 --- a/apps/sim/lib/api/contracts/knowledge/search.ts +++ b/apps/sim/lib/api/contracts/knowledge/search.ts @@ -1,5 +1,8 @@ import { z } from 'zod' -import { resolvedSecretTraceProvenanceSchema } from '@/lib/api/contracts/primitives' +import { + resolvedSecretTraceProvenanceSchema, + workspaceIdSchema, +} from '@/lib/api/contracts/primitives' import { defineRouteContract } from '@/lib/api/contracts/types' import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' import { DEFAULT_RERANKER_MODEL, rerankerModelSchema } from '@/lib/knowledge/reranker-models' @@ -16,15 +19,15 @@ export const knowledgeSearchTagFilterSchema = z.object({ export const KNOWLEDGE_SEARCH_MODES = ['vector', 'hybrid'] as const /** - * Shared by the internal and v1 search contracts. Defaults to `vector` so every - * existing caller keeps its current ranking; hybrid is opt-in. + * Shared by the internal and v1 search contracts. Omitted, the workspace's + * default applies: `hybrid` where permission-aware knowledge is on, else + * `vector`. The use case resolves that, so the schema carries no default. */ export const knowledgeSearchModeSchema = z .enum(KNOWLEDGE_SEARCH_MODES) .optional() .nullable() - .default('vector') - .transform((val) => val ?? 'vector') + .transform((val) => val ?? undefined) export const knowledgeSearchBodySchema = z .object({ @@ -51,9 +54,11 @@ export const knowledgeSearchBodySchema = z .nullable() .transform((val) => val || undefined), /** - * `vector` (default) is semantic-only retrieval. `hybrid` additionally runs a - * full-text leg and fuses the two by reciprocal rank, which recovers exact - * tokens (error codes, ticket keys, identifiers) that embeddings rank poorly. + * `hybrid` runs a full-text leg alongside semantic retrieval and fuses the + * two by reciprocal rank, which recovers exact tokens (error codes, ticket + * keys, identifiers) that embeddings rank poorly. `vector` is semantic-only + * retrieval. Omitted, the workspace's default applies. Where that default + * is `hybrid`, results in either mode also get a source-recency boost. */ searchMode: knowledgeSearchModeSchema, rerankerEnabled: z.boolean().optional().default(false), @@ -149,3 +154,52 @@ export const internalKnowledgeSearchContract = defineRouteContract({ }), }, }) + +/** One document a workspace search matched, with the best chunk of it. */ +export const workspaceKnowledgeSearchResultSchema = z.object({ + documentId: z.string(), + knowledgeBaseId: z.string(), + knowledgeBaseName: z.string(), + documentName: z.string().nullable(), + sourceUrl: z.string().nullable(), + connectorType: z.string().nullable(), + sourceModifiedAt: z.string().nullable(), + /** The person behind the document, from its author-like tag; null when the source names none. */ + author: z.string().nullable(), + content: z.string(), + chunkIndex: z.number(), + similarity: z.number(), +}) +export type WorkspaceKnowledgeSearchResult = z.output + +export const workspaceKnowledgeSearchBodySchema = z.object({ + workspaceId: workspaceIdSchema, + knowledgeBaseIds: z + .array(z.string().min(1, 'knowledgeBaseId cannot be empty')) + .min(1, 'At least one knowledge base is required') + .max(20, 'A search spans at most 20 knowledge bases'), + query: z.string().trim().min(1, 'A search query is required').max(2000, 'Query is too long'), + topK: z.number().int().min(1).max(50).optional().default(20), +}) +export type WorkspaceKnowledgeSearchBody = z.input + +/** + * The search a signed-in person runs from the composer: what their own + * account may read across the workspace's knowledge bases, presented as + * documents to open rather than chunks to feed a model. + */ +export const searchWorkspaceKnowledgeContract = defineRouteContract({ + method: 'POST', + path: '/api/knowledge/search', + body: workspaceKnowledgeSearchBodySchema, + response: { + mode: 'json', + schema: z.object({ + success: z.literal(true), + data: z.object({ + query: z.string(), + results: z.array(workspaceKnowledgeSearchResultSchema), + }), + }), + }, +}) diff --git a/apps/sim/lib/api/contracts/v1/knowledge/index.ts b/apps/sim/lib/api/contracts/v1/knowledge/index.ts index 2b13594b96a..412fa140c2d 100644 --- a/apps/sim/lib/api/contracts/v1/knowledge/index.ts +++ b/apps/sim/lib/api/contracts/v1/knowledge/index.ts @@ -182,8 +182,8 @@ export const v1KnowledgeSearchBodySchema = z topK: z.number().min(1).max(100).default(10), tagFilters: z.array(v1SearchTagFilterSchema).optional(), /** - * `vector` (default) is semantic-only retrieval; `hybrid` fuses a full-text - * leg with it by reciprocal rank. + * `hybrid` fuses a full-text leg with semantic retrieval by reciprocal + * rank; `vector` is semantic-only. Omitted, the workspace's default applies. */ searchMode: knowledgeSearchModeSchema, }) diff --git a/apps/sim/lib/api/contracts/workspaces.ts b/apps/sim/lib/api/contracts/workspaces.ts index 3b2c99e3a52..883c9375c0c 100644 --- a/apps/sim/lib/api/contracts/workspaces.ts +++ b/apps/sim/lib/api/contracts/workspaces.ts @@ -262,6 +262,8 @@ export const workspaceHostContextSchema = z.object({ features: z .object({ credentialGroups: z.boolean(), + /** Optional for rolling compatibility with app versions that predate the flag. */ + knowledgeMemberAccess: z.boolean().optional(), }) .optional(), }) diff --git a/apps/sim/lib/api/list-convention.test.ts b/apps/sim/lib/api/list-convention.test.ts index 255f6edd28b..ac7fa19028d 100644 --- a/apps/sim/lib/api/list-convention.test.ts +++ b/apps/sim/lib/api/list-convention.test.ts @@ -56,6 +56,7 @@ vi.mock('@/lib/workflows/skills/builtin-skills', () => ({ import { listVisibleWorkspaceCredentials } from '@/lib/credentials/queries' import { listFoldersForWorkspace } from '@/lib/folders/queries' +import { WORKSPACE_ACCESS_SCOPE } from '@/lib/knowledge/access/scope' import { getDocuments } from '@/lib/knowledge/documents/service' import { getWorkspaceKnowledgeBases } from '@/lib/knowledge/service' import { listWorkspaceMcpServers } from '@/lib/mcp/queries' @@ -182,7 +183,8 @@ const CASES: ListCase[] = [ getDocuments( 'knowledge-1', { search, sortBy: sortBy as never, sortOrder: sortOrder as never }, - 'request-1' + 'request-1', + WORKSPACE_ACCESS_SCOPE ), sort: { sortBy: 'fileSize', diff --git a/apps/sim/lib/atlassian/discovery.ts b/apps/sim/lib/atlassian/discovery.ts index c1a55718807..a57cdfe8b66 100644 --- a/apps/sim/lib/atlassian/discovery.ts +++ b/apps/sim/lib/atlassian/discovery.ts @@ -203,6 +203,26 @@ export function atlassianDiscoveryKey(resource: string, accessToken: string): st return `${resource}:${sha256Hex(accessToken).slice(0, 16)}` } +/** + * The credential reaches no Atlassian site at all. Typed so a caller acting + * for one person can tell "this person is not on the site" from a transport + * failure or a misconfigured domain. + */ +export class AtlassianSiteNotAccessibleError extends Error { + constructor(message: string) { + super(message) + this.name = 'AtlassianSiteNotAccessibleError' + } +} + +/** The credential reaches sites, but none matches the configured domain. */ +export class AtlassianSiteNotMatchedError extends Error { + constructor(message: string) { + super(message) + this.name = 'AtlassianSiteNotMatchedError' + } +} + /** * Picks the `cloudId` for `domain` out of an `accessible-resources` payload. * @@ -219,7 +239,7 @@ export function selectAtlassianCloudId( } if (resources.length === 0) { - throw new Error( + throw new AtlassianSiteNotAccessibleError( `No ${product} sites are accessible to this credential. ` + 'Reconnect the credential and grant access to the configured Atlassian site.' ) @@ -231,7 +251,7 @@ export function selectAtlassianCloudId( if (resources.length === 1) return resources[0].id - throw new Error( + throw new AtlassianSiteNotMatchedError( `Could not match ${product} domain "${domain}" to any accessible resource. ` + `Available sites: ${resources.map((r) => r.url).join(', ')}` ) diff --git a/apps/sim/lib/auth/connectors/managed-oauth.test.ts b/apps/sim/lib/auth/connectors/managed-oauth.test.ts index 6987f1498cb..c8053e6cd29 100644 --- a/apps/sim/lib/auth/connectors/managed-oauth.test.ts +++ b/apps/sim/lib/auth/connectors/managed-oauth.test.ts @@ -1,7 +1,8 @@ /** * @vitest-environment node */ -import { afterEach, describe, expect, it, vi } from 'vitest' +import { exportJWK, generateKeyPair, SignJWT } from 'jose' +import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest' import { createAtlassianManagedOAuthConnector, getManagedOAuthConnectorPolicy, @@ -395,3 +396,225 @@ describe('userinfo-backed managed OAuth connectors', () => { expect(identity.grantedScopes).toEqual(['data.records:read']) }) }) + +describe('Microsoft managed OAuth connector', () => { + const CLIENT_ID = 'client-1' + const TENANT_ID = 'tenant-1' + const MICROSOFT_PROVIDER_IDS = [ + 'microsoft-teams', + 'outlook', + 'onedrive', + 'sharepoint', + 'microsoft-excel', + ] + let privateKey: CryptoKey + let jwks: { keys: unknown[] } + + beforeAll(async () => { + const pair = await generateKeyPair('RS256', { extractable: true }) + privateKey = pair.privateKey + jwks = { + keys: [{ ...(await exportJWK(pair.publicKey)), kid: 'kid-1', use: 'sig', alg: 'RS256' }], + } + }) + + afterEach(() => { + vi.unstubAllGlobals() + }) + + function json(body: unknown): Response { + return new Response(JSON.stringify(body), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + } + + async function signIdToken( + claims: Record, + audience: string = CLIENT_ID + ): Promise { + return new SignJWT({ + oid: 'oid-1', + tid: TENANT_ID, + sub: 'pairwise-1', + email: 'person@example.com', + name: 'Person', + nonce: 'nonce-1', + ...claims, + }) + .setProtectedHeader({ alg: 'RS256', kid: 'kid-1' }) + .setIssuer(`https://login.microsoftonline.com/${TENANT_ID}/v2.0`) + .setAudience(audience) + .setIssuedAt() + .setExpirationTime('5m') + .sign(privateKey) + } + + function stubMicrosoft(userInfoSubject = 'pairwise-1'): ReturnType { + const fetchMock = vi.fn(async (input: string | URL | Request) => { + const url = String(input) + if (url.startsWith('https://login.microsoftonline.com/common/discovery/v2.0/keys')) { + return json(jwks) + } + if (url === 'https://graph.microsoft.com/oidc/userinfo') return json({ sub: userInfoSubject }) + throw new Error(`Unexpected fetch: ${url}`) + }) + vi.stubGlobal('fetch', fetchMock) + return fetchMock + } + + function policyFor(providerId: string) { + const policy = getManagedOAuthConnectorPolicy(providerId) + if (!policy) throw new Error(`No managed OAuth policy registered for ${providerId}`) + return policy + } + + it('governs every Microsoft provider through one app registration', () => { + const appIds = new Set( + MICROSOFT_PROVIDER_IDS.map((providerId) => { + const policy = policyFor(providerId) + expect(policy).toMatchObject({ + requiresRefreshToken: true, + pkce: true, + nonceVerification: 'id_token', + includeLoginHint: true, + prompt: 'select_account', + }) + return policy.getAuthorizationAppId(CLIENT_ID) + }) + ) + expect(appIds.size).toBe(1) + expect([...appIds][0]).toMatch(/^microsoft:[0-9a-f]{64}$/) + expect(getManagedOAuthConnectorPolicy('microsoft-word')).toBeUndefined() + }) + + it('verifies the id token, binds the access token to it, and reports what Entra proves', async () => { + const fetchMock = stubMicrosoft() + + const identity = await policyFor('onedrive').verifyIdentity({ + tokens: { + tokenType: 'Bearer', + accessToken: 'access-1', + refreshToken: 'refresh-1', + idToken: await signIdToken({}), + scopes: ['Files.Read', 'User.Read'], + }, + clientId: CLIENT_ID, + }) + + expect(identity).toMatchObject({ + providerSubjectId: 'oid-1', + providerTenantId: TENANT_ID, + email: 'person@example.com', + emailVerified: false, + displayName: 'Person', + nonce: 'nonce-1', + }) + expect([...identity.grantedScopes].sort()).toEqual( + ['Files.Read', 'User.Read', 'email', 'offline_access', 'openid', 'profile'].sort() + ) + expect(fetchMock).toHaveBeenCalledWith( + 'https://graph.microsoft.com/oidc/userinfo', + expect.objectContaining({ + headers: expect.objectContaining({ Authorization: 'Bearer access-1' }), + }) + ) + }) + + it.each([ + ['xms_edov', { xms_edov: true }], + ['email_verified', { email_verified: true }], + ])('counts the email verified when Entra asserts it through %s', async (_claim, claims) => { + stubMicrosoft() + + const identity = await policyFor('outlook').verifyIdentity({ + tokens: { tokenType: 'Bearer', accessToken: 'access-1', idToken: await signIdToken(claims) }, + clientId: CLIENT_ID, + }) + + expect(identity.emailVerified).toBe(true) + }) + + it('does not count offline access as granted without a refresh token', async () => { + stubMicrosoft() + + const identity = await policyFor('sharepoint').verifyIdentity({ + tokens: { tokenType: 'Bearer', accessToken: 'access-1', idToken: await signIdToken({}) }, + clientId: CLIENT_ID, + }) + + expect(identity.grantedScopes).not.toContain('offline_access') + }) + + it('rejects an access token that resolves to another subject', async () => { + stubMicrosoft('pairwise-2') + + await expect( + policyFor('microsoft-teams').verifyIdentity({ + tokens: { tokenType: 'Bearer', accessToken: 'access-1', idToken: await signIdToken({}) }, + clientId: CLIENT_ID, + }) + ).rejects.toThrow('Microsoft returned an access token for another identity') + }) + + it('rejects an id token issued for another client', async () => { + stubMicrosoft() + + await expect( + policyFor('microsoft-excel').verifyIdentity({ + tokens: { + tokenType: 'Bearer', + accessToken: 'access-1', + idToken: await signIdToken({}, 'client-2'), + }, + clientId: CLIENT_ID, + }) + ).rejects.toThrow() + }) + + it.each([ + ['object id', { oid: undefined }], + ['tenant id', { tid: undefined }], + ['issuer of its own tenant', { tid: 'tenant-2' }], + ])('rejects an id token without the %s', async (_label, claims) => { + stubMicrosoft() + + await expect( + policyFor('onedrive').verifyIdentity({ + tokens: { + tokenType: 'Bearer', + accessToken: 'access-1', + idToken: await signIdToken(claims), + }, + clientId: CLIENT_ID, + }) + ).rejects.toThrow('Microsoft returned an invalid identity token') + }) + + it('rejects an id token that names no email to bind the invitation to', async () => { + stubMicrosoft() + + await expect( + policyFor('onedrive').verifyIdentity({ + tokens: { + tokenType: 'Bearer', + accessToken: 'access-1', + idToken: await signIdToken({ email: undefined }), + }, + clientId: CLIENT_ID, + }) + ).rejects.toThrow('Microsoft returned an identity token without an email') + }) + + it('compares scopes by name regardless of case or resource prefix', () => { + const policy = policyFor('onedrive') + + expect( + policy.hasRequiredScopes( + ['https://graph.microsoft.com/Files.Read', 'MAIL.READ', 'offline_access'], + ['files.read', 'Mail.Read'] + ) + ).toBe(true) + expect(policy.hasRequiredScopes(['Files.Read'], ['Files.ReadWrite'])).toBe(false) + }) +}) diff --git a/apps/sim/lib/auth/connectors/managed-oauth.ts b/apps/sim/lib/auth/connectors/managed-oauth.ts index d43d9cf56e3..8d63e37de79 100644 --- a/apps/sim/lib/auth/connectors/managed-oauth.ts +++ b/apps/sim/lib/auth/connectors/managed-oauth.ts @@ -3,9 +3,11 @@ import { isRecordLike } from '@sim/utils/object' import type { OAuth2Tokens } from 'better-auth/oauth2' import type { GenericOAuthConfig } from 'better-auth/plugins' import { OAuth2Client, type TokenPayload } from 'google-auth-library' +import { createRemoteJWKSet, type JWTPayload, jwtVerify } from 'jose' import { buildConnectorProviders } from '@/lib/auth/connectors/providers' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' import { getDocusignOAuthUrl } from '@/lib/oauth/docusign' +import { deriveMicrosoftEmailVerified, mapMicrosoftProfileToUser } from '@/lib/oauth/microsoft' import { SALESFORCE_LOGIN_HOSTS } from '@/lib/oauth/salesforce' import { isTerminalRefreshError } from '@/lib/oauth/terminal-errors' import { getCanonicalScopesForProvider } from '@/lib/oauth/utils' @@ -21,6 +23,19 @@ const GMAIL_LABELS_SCOPE = 'https://www.googleapis.com/auth/gmail.labels' const ATLASSIAN_USER_INFO_URL = 'https://api.atlassian.com/me' const ATLASSIAN_USER_INFO_MAX_BYTES = 256 * 1024 const ATLASSIAN_USER_INFO_TIMEOUT_MS = 10_000 +const MICROSOFT_JWKS_URL = 'https://login.microsoftonline.com/common/discovery/v2.0/keys' +const MICROSOFT_OIDC_USER_INFO_URL = 'https://graph.microsoft.com/oidc/userinfo' +const MICROSOFT_OIDC_USER_INFO_MAX_BYTES = 256 * 1024 +const MICROSOFT_OIDC_USER_INFO_TIMEOUT_MS = 10_000 +const MICROSOFT_GRAPH_SCOPE_PREFIX = 'https://graph.microsoft.com/' +/** The Microsoft providers whose accounts a Credential Group can collect per person. */ +const MICROSOFT_MANAGED_OAUTH_PROVIDER_IDS = new Set([ + 'microsoft-teams', + 'outlook', + 'onedrive', + 'sharepoint', + 'microsoft-excel', +]) type AtlassianManagedOAuthProviderId = 'confluence' | 'jira' @@ -227,6 +242,160 @@ export function createAtlassianManagedOAuthConnector( } } +let microsoftJwks: ReturnType | undefined + +/** The signing keys of the multi-tenant Microsoft identity platform, cached across verifications. */ +function getMicrosoftJwks(): ReturnType { + microsoftJwks ??= createRemoteJWKSet(new URL(MICROSOFT_JWKS_URL)) + return microsoftJwks +} + +/** + * Graph delegated scopes compare by name regardless of case, and a token response may spell + * one as its resource-qualified form. + */ +function canonicalMicrosoftScope(scope: string): string { + const unqualified = scope.startsWith(MICROSOFT_GRAPH_SCOPE_PREFIX) + ? scope.slice(MICROSOFT_GRAPH_SCOPE_PREFIX.length) + : scope + return unqualified.toLowerCase() +} + +interface MicrosoftIdentityClaims { + oid: string + tid: string + sub: string + email: string + name?: string + nonce?: string + claims: Record +} + +/** + * Reads the claims a verified Microsoft id_token must carry to bind an enrollment. `oid` and + * `tid` identify the person and their tenant stably across every Microsoft app; `sub` is the + * app-pairwise subject the OIDC userinfo endpoint echoes back. The issuer is checked against the + * token's own tenant because the multi-tenant `/common` authority signs for every tenant. + */ +function requireMicrosoftIdentityClaims(payload: JWTPayload): MicrosoftIdentityClaims { + const claims: Record = { ...payload } + const { oid, tid, sub, iss, name, nonce } = claims + if ( + typeof oid !== 'string' || + !oid || + typeof tid !== 'string' || + !tid || + typeof sub !== 'string' || + !sub || + iss !== `https://login.microsoftonline.com/${tid}/v2.0` + ) { + throw new Error('Microsoft returned an invalid identity token') + } + const email = [claims.email, claims.preferred_username, claims.upn].find( + (value): value is string => typeof value === 'string' && value.trim().length > 0 + ) + if (!email) { + throw new Error('Microsoft returned an identity token without an email') + } + return { + oid, + tid, + sub, + email, + ...(typeof name === 'string' && name.trim() ? { name } : {}), + ...(typeof nonce === 'string' && nonce ? { nonce } : {}), + claims, + } +} + +/** + * The subject the access token resolves to at Microsoft's OIDC userinfo endpoint. It needs only + * the `openid` grant, so unlike Graph `/me` it does not fail for a tenant whose administrator has + * not consented to Graph. + */ +async function fetchMicrosoftAccessTokenSubject(accessToken: string): Promise { + const response = await fetch(MICROSOFT_OIDC_USER_INFO_URL, { + headers: { Accept: 'application/json', Authorization: `Bearer ${accessToken}` }, + signal: AbortSignal.timeout(MICROSOFT_OIDC_USER_INFO_TIMEOUT_MS), + }) + const profile = await readResponseJsonWithLimit(response, { + maxBytes: MICROSOFT_OIDC_USER_INFO_MAX_BYTES, + label: 'Microsoft user identity response', + }) + if (!response.ok) { + throw new Error(`Microsoft user identity request failed with HTTP ${response.status}`) + } + if (!isRecordLike(profile) || typeof profile.sub !== 'string' || !profile.sub) { + throw new Error('Microsoft returned an invalid user identity') + } + return profile.sub +} + +/** + * Managed enrollment policy for the providers that share Sim's Microsoft app registration. + * + * Identity comes from the id_token, verified against the identity platform's published keys and + * bound to the access token through the OIDC userinfo subject, the way the Google policy binds + * through tokeninfo. Microsoft never asserts `email_verified` for a work account, so the email + * counts as proven only through the claims Entra does vouch for: the verified-email claims, or + * `xms_edov` asserting the domain belongs to the account's own tenant. + */ +export function createMicrosoftManagedOAuthConnector( + providerId: string +): ManagedOAuthConnectorConfig { + return { + additionalScopes: [], + requiresRefreshToken: true, + pkce: true, + nonceVerification: 'id_token', + includeLoginHint: true, + prompt: 'select_account', + getAuthorizationAppId(clientId) { + return `microsoft:${createHash('sha256').update(clientId).digest('hex')}` + }, + async verifyIdentity({ tokens, clientId }) { + if (!tokens.idToken || !tokens.accessToken) { + throw new Error(`Microsoft ${providerId} returned an incomplete authorization`) + } + const { payload } = await jwtVerify(tokens.idToken, getMicrosoftJwks(), { + audience: clientId, + }) + const identity = requireMicrosoftIdentityClaims(payload) + const accessTokenSubject = await fetchMicrosoftAccessTokenSubject(tokens.accessToken) + if (accessTokenSubject !== identity.sub) { + throw new Error('Microsoft returned an access token for another identity') + } + /** + * The token response's `scope` is not guaranteed to echo the OIDC scopes or + * `offline_access`, so each is counted only when the response itself proves the grant: an + * id_token for `openid`, its `name` and `email` claims for `profile` and `email`, and a + * refresh token for `offline_access`. + */ + const grantedScopes = new Set(tokens.scopes ?? []) + grantedScopes.add('openid') + if (identity.name) grantedScopes.add('profile') + if (typeof identity.claims.email === 'string') grantedScopes.add('email') + if (tokens.refreshToken) grantedScopes.add('offline_access') + return { + providerSubjectId: identity.oid, + providerTenantId: identity.tid, + email: identity.email, + emailVerified: + deriveMicrosoftEmailVerified(identity.claims, identity.email) || + mapMicrosoftProfileToUser(identity.claims).emailVerified === true, + ...(identity.name ? { displayName: identity.name } : {}), + ...(identity.nonce ? { nonce: identity.nonce } : {}), + grantedScopes: [...grantedScopes], + } + }, + hasRequiredScopes(grantedScopes, requiredScopes) { + const granted = new Set(grantedScopes.map(canonicalMicrosoftScope)) + return requiredScopes.every((scope) => granted.has(canonicalMicrosoftScope(scope))) + }, + isTerminalRefreshError, + } +} + const USER_INFO_TIMEOUT_MS = 10_000 const USER_INFO_MAX_BYTES = 256 * 1024 @@ -559,6 +728,92 @@ function createAttioManagedOAuthConnector(): ManagedOAuthConnectorConfig { } } +const BITBUCKET_API_BASE = 'https://api.bitbucket.org/2.0' +const BITBUCKET_EMAIL_SCOPE = 'email' + +/** + * Bitbucket's current-user endpoint carries no address, and its emails endpoint needs the + * `email` scope the consumer would not otherwise request; so this policy adds that scope and + * reads the two resources in turn. The subject is the immutable `account_id`; there is no + * tenant because one Bitbucket account belongs to any number of workspaces. + */ +function createBitbucketManagedOAuthConnector(): ManagedOAuthConnectorConfig { + return { + additionalScopes: [BITBUCKET_EMAIL_SCOPE], + requiresRefreshToken: true, + pkce: false, + nonceVerification: 'state_only', + includeLoginHint: false, + getAuthorizationAppId(clientId) { + return `bitbucket:${createHash('sha256').update(clientId).digest('hex')}` + }, + async verifyIdentity({ tokens }) { + if (!tokens.accessToken) { + throw new Error('Bitbucket returned an incomplete authorization') + } + const headers = { + Accept: 'application/json', + Authorization: `Bearer ${tokens.accessToken}`, + } + const userResponse = await fetch(`${BITBUCKET_API_BASE}/user`, { + headers, + signal: AbortSignal.timeout(USER_INFO_TIMEOUT_MS), + }) + const userBody = await readResponseJsonWithLimit(userResponse, { + maxBytes: USER_INFO_MAX_BYTES, + label: 'Bitbucket identity response', + }) + if (!userResponse.ok) { + throw new Error(`Bitbucket identity request failed with HTTP ${userResponse.status}`) + } + const user = asProfileRecord(userBody, 'Bitbucket') + const accountId = requireIdentityField(user.account_id, 'Bitbucket account id') + const emailsResponse = await fetch(`${BITBUCKET_API_BASE}/user/emails`, { + headers, + signal: AbortSignal.timeout(USER_INFO_TIMEOUT_MS), + }) + const emailsBody = await readResponseJsonWithLimit(emailsResponse, { + maxBytes: USER_INFO_MAX_BYTES, + label: 'Bitbucket emails response', + }) + if (!emailsResponse.ok) { + throw new Error(`Bitbucket emails request failed with HTTP ${emailsResponse.status}`) + } + const emails = asProfileRecord(emailsBody, 'Bitbucket').values + const primary = Array.isArray(emails) + ? emails.find( + (entry): entry is Record => + isRecordLike(entry) && entry.is_primary === true && entry.is_confirmed === true + ) + : undefined + const avatar = + isRecordLike(user.links) && isRecordLike(user.links.avatar) + ? user.links.avatar.href + : undefined + const base = withOptionalIdentityFields( + { + providerSubjectId: accountId, + email: requireIdentityField(primary?.email, 'Bitbucket confirmed primary email'), + /** Only a confirmed primary address is accepted above. */ + emailVerified: true, + }, + { displayName: user.display_name, avatarUrl: avatar } + ) + return { + ...base, + providerTenantId: null, + /** Bitbucket reports the consumer's granted scopes on the token response. */ + grantedScopes: [...new Set(tokens.scopes ?? [])], + } + }, + hasRequiredScopes(grantedScopes, requiredScopes) { + const granted = new Set(grantedScopes) + return requiredScopes.every((scope) => granted.has(scope)) + }, + isTerminalRefreshError, + } +} + /** * Managed enrollment policies for the providers whose identity endpoint reports an email the * provider itself vouches for. Keyed by connector provider id. @@ -719,6 +974,7 @@ const USER_INFO_MANAGED_OAUTH_CONNECTORS = new Map ManagedOAuthCon }), ], ['attio', createAttioManagedOAuthConnector], + ['bitbucket', createBitbucketManagedOAuthConnector], [ 'hubspot', () => @@ -1031,12 +1287,24 @@ export function getManagedOAuthConnectorPolicy( function resolveManagedOAuthPolicy( providerId: string ): (() => ManagedOAuthConnectorConfig) | undefined { - if (providerId === 'google-email' || providerId === 'google-calendar') { + if ( + providerId === 'google-email' || + providerId === 'google-calendar' || + providerId === 'google-drive' || + providerId === 'google-docs' || + providerId === 'google-forms' || + providerId === 'google-chat' || + providerId === 'google-meet' || + providerId === 'google-sheets' + ) { return () => createGoogleManagedOAuthConnector(providerId) } if (providerId === 'confluence' || providerId === 'jira') { return () => createAtlassianManagedOAuthConnector(providerId) } + if (MICROSOFT_MANAGED_OAUTH_PROVIDER_IDS.has(providerId)) { + return () => createMicrosoftManagedOAuthConnector(providerId) + } return USER_INFO_MANAGED_OAUTH_CONNECTORS.get(providerId) } diff --git a/apps/sim/lib/copilot/auth/application-delegation.ts b/apps/sim/lib/copilot/auth/application-delegation.ts index 4a779e0a021..e320ab71778 100644 --- a/apps/sim/lib/copilot/auth/application-delegation.ts +++ b/apps/sim/lib/copilot/auth/application-delegation.ts @@ -34,7 +34,7 @@ export class InteractiveCopilotExecutionRequiredError extends Error { export type CopilotResourceScope = Pick< NonNullable, - 'fileId' | 'tableId' + 'fileId' | 'tableId' | 'credentialId' > export interface CopilotDelegationConfiguration { @@ -125,11 +125,17 @@ export function createTrustedCopilotPrincipal( if (options.resourceScope?.tableId !== undefined) { requireNonEmpty(options.resourceScope.tableId, 'a valid table scope') } + if (options.resourceScope?.credentialId !== undefined) { + requireNonEmpty(options.resourceScope.credentialId, 'a valid credential scope') + } const issuedAt = new Date() const resourceScope = Object.freeze({ ...(options.resourceScope?.fileId ? { fileId: options.resourceScope.fileId } : {}), ...(options.resourceScope?.tableId ? { tableId: options.resourceScope.tableId } : {}), + ...(options.resourceScope?.credentialId + ? { credentialId: options.resourceScope.credentialId } + : {}), ...(input.chatId ? { chatId: input.chatId } : {}), ...(input.executionId ? { executionId: input.executionId } : {}), }) diff --git a/apps/sim/lib/copilot/chat/ask-mode.test.ts b/apps/sim/lib/copilot/chat/ask-mode.test.ts new file mode 100644 index 00000000000..e18f4271837 --- /dev/null +++ b/apps/sim/lib/copilot/chat/ask-mode.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from 'vitest' +import { ASK_MODE_AGENT_CONTEXT, withAskModeContext } from '@/lib/copilot/chat/ask-mode' + +const knowledge = { type: 'knowledge', content: '', tag: '@Sim Search', path: 'knowledge/x.json' } + +describe('withAskModeContext', () => { + it('appends the Ask skill after the attached contexts on an Ask turn', () => { + expect(withAskModeContext([knowledge], 'ask')).toEqual([knowledge, ASK_MODE_AGENT_CONTEXT]) + }) + + it('leaves every other mode untouched', () => { + for (const mode of ['agent', 'build', 'plan', undefined]) { + expect(withAskModeContext([knowledge], mode)).toEqual([knowledge]) + } + }) + + it('renders as a skill the agent injects for the turn', () => { + expect(ASK_MODE_AGENT_CONTEXT.type).toBe('skill') + expect(ASK_MODE_AGENT_CONTEXT.content).toContain('') + expect(ASK_MODE_AGENT_CONTEXT.content).toContain('query') + }) +}) diff --git a/apps/sim/lib/copilot/chat/ask-mode.ts b/apps/sim/lib/copilot/chat/ask-mode.ts new file mode 100644 index 00000000000..02d7b81232b --- /dev/null +++ b/apps/sim/lib/copilot/chat/ask-mode.ts @@ -0,0 +1,37 @@ +/** A context item as the agent receives it; skills carry their instructions in `content`. */ +export interface AskModeAgentContext { + type: 'skill' + tag: string + content: string +} + +/** The request mode of an Ask turn, as the composer sends it. */ +export const ASK_REQUEST_MODE = 'ask' + +/** + * The instructions an Ask turn carries. Rendered by the agent as an active + * skill for the turn, alongside the knowledge bases the composer attached, so + * the model searches them first and answers with citations, reaching a + * connected service only when the indexed sources cannot answer. + */ +export const ASK_MODE_AGENT_CONTEXT: AskModeAgentContext = { + type: 'skill', + tag: '@Ask', + content: [ + 'The person chose the Assistant: they want an answer in natural language drawn from their connected sources, not an action.', + '', + '- Answer from the knowledge bases attached to this message first. Search them with the knowledge tool `query` operation, and search again with other phrasings when the first pass returns little. Do not read a base or its metadata first; search.', + "- Reach for a connected integration only when the indexed sources cannot answer: live or very recent data (today's inbox, a calendar), or an action the person asked for outright. Say which service you used. Never build, run, or schedule anything on an Assistant turn.", + '- Cite every claim with a `` tag exactly as the knowledge tool describes. When nothing relevant is found, say so plainly instead of guessing.', + '- Keep the answer short: lead with the answer, then the supporting points.', + '- Suggested follow-ups, when you offer them, are questions the attached sources can answer. Never suggest building, running, or automating anything.', + ].join('\n'), +} + +/** The turn's contexts with the Ask instructions appended when the request asked for Ask mode. */ +export function withAskModeContext( + contexts: T[], + mode: string | undefined +): Array { + return mode === ASK_REQUEST_MODE ? [...contexts, ASK_MODE_AGENT_CONTEXT] : contexts +} diff --git a/apps/sim/lib/copilot/chat/payload.ts b/apps/sim/lib/copilot/chat/payload.ts index 795cc1f7f36..cec2b962bfe 100644 --- a/apps/sim/lib/copilot/chat/payload.ts +++ b/apps/sim/lib/copilot/chat/payload.ts @@ -412,8 +412,10 @@ export async function buildCopilotRequestPayload( const payloadLogger = logger.withMetadata({ messageId: userMessageId }) // "superagent" is a legacy wire value for Direct Action mode; both modes - // execute connected-service operations through the main-agent gateway. - if (effectiveMode === 'build' || effectiveMode === 'superagent') { + // execute connected-service operations through the main-agent gateway. An + // Ask turn keeps them too: it answers from knowledge first and reaches a + // connected service only when the indexed sources cannot answer. + if (effectiveMode === 'build' || effectiveMode === 'superagent' || effectiveMode === 'ask') { integrationTools = await buildIntegrationToolSchemas( userId, userMessageId, diff --git a/apps/sim/lib/copilot/chat/post.ts b/apps/sim/lib/copilot/chat/post.ts index 112efea4011..e19cb5ea417 100644 --- a/apps/sim/lib/copilot/chat/post.ts +++ b/apps/sim/lib/copilot/chat/post.ts @@ -11,6 +11,7 @@ import { isZodError, validationErrorResponse } from '@/lib/api/server' import { getSession } from '@/lib/auth' import { resolveBillingAttribution } from '@/lib/billing/core/billing-attribution' import { chatOperations } from '@/lib/copilot/application/operations' +import { withAskModeContext } from '@/lib/copilot/chat/ask-mode' import { DESKTOP_TERMINAL_HINT_ID_MAX_LENGTH, DESKTOP_TERMINAL_HINT_TEXT_MAX_LENGTH, @@ -934,7 +935,7 @@ async function resolveBranch(params: { workspaceId: requestedWorkspaceId, userId: payloadParams.userId, userMessageId: payloadParams.userMessageId, - mode: 'agent', + mode: mode ?? 'agent', model: '', contexts: payloadParams.contexts, mcpServerIds: payloadParams.mcpServerIds, @@ -961,7 +962,7 @@ async function resolveBranch(params: { chatId, messageId, userTimezone, - requestMode: 'agent', + requestMode: mode ?? 'agent', }), } } @@ -1374,6 +1375,7 @@ export async function handleUnifiedChatPost(req: NextRequest) { // typed snapshot Go diffs into baseline+delta messages. const workspaceContext = workspaceSnapshot?.markdown const vfs = workspaceSnapshot?.snapshot + const turnContexts = withAskModeContext(agentContexts, body.mode) executionContext.userPermission = userPermission ?? undefined @@ -1397,7 +1399,7 @@ export async function handleUnifiedChatPost(req: NextRequest) { userId: authenticatedUserId, userMessageId, chatId: actualChatId, - contexts: agentContexts, + contexts: turnContexts, mcpServerIds, fileAttachments: body.fileAttachments, userPermission: userPermission ?? undefined, @@ -1425,7 +1427,7 @@ export async function handleUnifiedChatPost(req: NextRequest) { userId: authenticatedUserId, userMessageId, chatId: actualChatId, - contexts: agentContexts, + contexts: turnContexts, mcpServerIds, fileAttachments: body.fileAttachments, userPermission: userPermission ?? undefined, diff --git a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts index c65ad755b66..db952b6a830 100644 --- a/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts +++ b/apps/sim/lib/copilot/tools/server/knowledge/knowledge-base.ts @@ -52,11 +52,22 @@ import { KNOWLEDGE_TAG_DISPLAY_NAME_MAX_LENGTH, MAX_KNOWLEDGE_BATCH_ITEMS, } from '@/lib/knowledge/constants' +import { sourceAuthor } from '@/lib/knowledge/search/author' import { captureServerEvent } from '@/lib/posthog/server' import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-secret-content-projection' const logger = createLogger('KnowledgeBaseServerTool') +/** Results a query returns unless the caller asks for a number. */ +const DEFAULT_QUERY_TOP_K = 5 +/** + * How the model cites a knowledge result in its reply. The `` tag is + * what the chat renders as a link back to the document, so a result without + * a source URL is quoted by name instead. + */ +const KNOWLEDGE_CITATION_INSTRUCTION = + 'Cite each result you use inline, right after the sentence it supports, as {"url":"","title":"","siteName":"","connectorType":"","snippet":"","updatedAt":"","author":""} with every value JSON-escaped; leave out any optional field whose value is null or unknown, and omit the tag for a result whose sourceUrl is null and name the document instead.' + /** * Resolves an environment-variable reference passed as a connector API key. * @@ -405,7 +416,7 @@ export const knowledgeBaseServerTool: BaseServerTool ({ documentId: result.documentId, + documentName: result.documentName, + sourceUrl: result.sourceUrl, + sourceModifiedAt: result.sourceModifiedAt?.toISOString() ?? null, + author: sourceAuthor(result.metadata), + connectorType: result.connectorType, content: result.content, chunkIndex: result.chunkIndex, similarity: result.similarity, diff --git a/apps/sim/lib/copilot/tools/tool-display.test.ts b/apps/sim/lib/copilot/tools/tool-display.test.ts index d3e6e212d4f..ef8257efdc7 100644 --- a/apps/sim/lib/copilot/tools/tool-display.test.ts +++ b/apps/sim/lib/copilot/tools/tool-display.test.ts @@ -378,6 +378,12 @@ describe('getToolDisplayTitle for operation-driven tools', () => { expect(getToolDisplayTitle('manage_knowledge_base', { operation: 'query' })).toBe( 'Searching knowledge base' ) + expect( + getToolDisplayTitle('manage_knowledge_base', { + operation: 'query', + args: { query: 'volvo delivery process' }, + }) + ).toBe('Searching knowledge base for volvo delivery process') expect(getToolDisplayTitle('manage_knowledge_base', { operation: 'sync_connector' })).toBe( 'Syncing knowledge base connector' ) diff --git a/apps/sim/lib/copilot/tools/tool-display.ts b/apps/sim/lib/copilot/tools/tool-display.ts index 9cc094d81c5..1633f7bbe68 100644 --- a/apps/sim/lib/copilot/tools/tool-display.ts +++ b/apps/sim/lib/copilot/tools/tool-display.ts @@ -301,10 +301,11 @@ function knowledgeBaseTitle(args: ToolArgs): string { 'file' ) + const query = stringArg(operationArgs, 'query') const titles: Record = { create: `Creating ${name || 'knowledge base'}`, get: 'Reading knowledge base', - query: 'Searching knowledge base', + query: query ? `Searching knowledge base for ${query}` : 'Searching knowledge base', add_file: `Adding ${fileTarget} to knowledge base`, update: 'Updating knowledge base', delete: `Deleting ${countedResourceTarget(operationArgs, 'knowledgeBaseIds', 'knowledge base', 'knowledge bases')}`, diff --git a/apps/sim/lib/copilot/vfs/serializers.ts b/apps/sim/lib/copilot/vfs/serializers.ts index 647684e7e78..cf555ba62a1 100644 --- a/apps/sim/lib/copilot/vfs/serializers.ts +++ b/apps/sim/lib/copilot/vfs/serializers.ts @@ -769,8 +769,12 @@ export function serializeCredentials( description?: string | null role?: string | null scope: string | null - /** 'service_account' for a shared app credential; omitted/undefined for a personal OAuth connection. */ - credentialType?: 'oauth' | 'service_account' + /** + * 'service_account' for a shared app credential, 'managed_oauth' for a + * Credential Group credential the person holds through their enrollment; + * omitted/undefined for a personal OAuth connection. + */ + credentialType?: 'oauth' | 'service_account' | 'managed_oauth' createdAt: Date }> ): string { @@ -783,6 +787,7 @@ export function serializeCredentials( role: a.role || undefined, scope: a.scope || undefined, // 'oauth' (personal connection) vs 'service_account' (shared app + // credential) vs 'managed_oauth' (the person's own Credential Group // credential) — they reconnect differently, so the agent must branch on // this. Env-var credentials carry no type. type: a.credentialType, diff --git a/apps/sim/lib/copilot/vfs/workspace-vfs.ts b/apps/sim/lib/copilot/vfs/workspace-vfs.ts index 5e2d841568d..eec6cca7c2a 100644 --- a/apps/sim/lib/copilot/vfs/workspace-vfs.ts +++ b/apps/sim/lib/copilot/vfs/workspace-vfs.ts @@ -120,6 +120,7 @@ import { listCredentialGroups } from '@/lib/credential-groups/service' import { getAccessibleEnvCredentials, getAccessibleOAuthCredentials, + getEnrolledManagedOAuthCredentials, } from '@/lib/credentials/environment' import { getPersonalAndWorkspaceEnv } from '@/lib/environment/utils' import { BINARY_DOC_TASKS, MAX_DOCUMENT_PREVIEW_CODE_BYTES } from '@/lib/execution/constants' @@ -3174,7 +3175,12 @@ export class WorkspaceVFS { const [envCredentials, oauthCredentials, apiKeyRows, envData, permissionConfig] = await Promise.all([ getAccessibleEnvCredentials(workspaceId, userId, { isWorkspaceAdmin }), - getAccessibleOAuthCredentials(workspaceId, userId, { isWorkspaceAdmin }), + getAccessibleOAuthCredentials(workspaceId, userId, { isWorkspaceAdmin }).then( + async (accessible) => [ + ...accessible, + ...(await getEnrolledManagedOAuthCredentials(workspaceId, userId)), + ] + ), listApiKeys(workspaceId), getPersonalAndWorkspaceEnv(userId, workspaceId), permissionConfigPromise, diff --git a/apps/sim/lib/core/application/index.ts b/apps/sim/lib/core/application/index.ts index 52cecb7dbb4..75d6ba3ce2d 100644 --- a/apps/sim/lib/core/application/index.ts +++ b/apps/sim/lib/core/application/index.ts @@ -42,6 +42,7 @@ export { PersonalApiKeysDisabledError, PrincipalKindAuthorizationError, requireAllowedWorkspacePrincipal, + requireCurrentHumanRole, requirePersonalApiKeysAllowed, WorkspaceApiKeyAuthorizationError, WorkspaceApiKeyScopeAuthorizationError, diff --git a/apps/sim/lib/core/application/workspace-authorization.ts b/apps/sim/lib/core/application/workspace-authorization.ts index 167a7972234..2b834162911 100644 --- a/apps/sim/lib/core/application/workspace-authorization.ts +++ b/apps/sim/lib/core/application/workspace-authorization.ts @@ -248,7 +248,7 @@ export async function requirePersonalApiKeysAllowed( * raising a role, rather than chasing an admin about a group setting that is * not why they were refused. */ -async function requireCurrentHumanRole( +export async function requireCurrentHumanRole( userId: string, context: C, required: PermissionType, @@ -264,6 +264,13 @@ async function requireCurrentHumanRole( requirePermission(permission, required) } +/** + * A use case's own escalation: refuses unless the person holds `required` in + * the workspace right now. For an operation whose minimum role fits most of + * its inputs but one variant needs more — a connector that crawls as every + * enrolled member is an admin decision even though creating a connector is + * not — so the operation keeps its role and the variant asserts its own. + */ async function requireCurrentHumanAccess( userId: string, context: C, diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index 833ad743b79..bd28ef56842 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -589,6 +589,7 @@ export const env = createEnv({ TABLES_V2_API: z.boolean().optional(), // Enable the v2 tables HTTP API (public /api/v2/tables + internal /api/table/[tableId]/query predicate-grammar route) TABLE_ROW_TTL: z.boolean().optional(), CREDENTIAL_GROUPS: z.boolean().optional(), // Enable enterprise Credential Groups globally + KNOWLEDGE_MEMBER_ACCESS: z.boolean().optional(), // Enable per-member knowledge connectors and hybrid-by-default retrieval globally // Organizations - for self-hosted deployments ORGANIZATIONS_ENABLED: z.boolean().optional(), // Enable organizations on self-hosted (bypasses plan requirements) diff --git a/apps/sim/lib/core/config/feature-flags.test.ts b/apps/sim/lib/core/config/feature-flags.test.ts index fefd67589a3..2d28ffa4cf4 100644 --- a/apps/sim/lib/core/config/feature-flags.test.ts +++ b/apps/sim/lib/core/config/feature-flags.test.ts @@ -14,6 +14,7 @@ const { mockFetch, mockIsPlatformAdmin, envRef } = vi.hoisted(() => ({ TABLES_V2_API: undefined as boolean | undefined, TABLE_ROW_TTL: undefined as boolean | undefined, CREDENTIAL_GROUPS: undefined as boolean | undefined, + KNOWLEDGE_MEMBER_ACCESS: undefined as boolean | undefined, }, })) @@ -123,6 +124,40 @@ describe('isFeatureEnabled', () => { vi.clearAllMocks() setEnvFlags({ isAppConfigEnabled: false }) envRef.CREDENTIAL_GROUPS = undefined + envRef.KNOWLEDGE_MEMBER_ACCESS = undefined + }) + + describe('knowledge-member-access flag', () => { + it('uses a global fallback switch off AppConfig', async () => { + expect(await isFeatureEnabled('knowledge-member-access')).toBe(false) + + envRef.KNOWLEDGE_MEMBER_ACCESS = true + expect(await isFeatureEnabled('knowledge-member-access')).toBe(true) + }) + + it('opens for an allowlisted workspace only', async () => { + withAppConfig({ 'knowledge-member-access': { workspaceIds: ['ws-1'] } }) + expect( + await isFeatureEnabled('knowledge-member-access', { workspaceId: 'ws-1', userId: 'u1' }) + ).toBe(true) + expect( + await isFeatureEnabled('knowledge-member-access', { workspaceId: 'ws-2', userId: 'u1' }) + ).toBe(false) + expect(mockIsPlatformAdmin).not.toHaveBeenCalled() + }) + + it('opens for a platform admin in any workspace', async () => { + withAppConfig({ 'knowledge-member-access': { workspaceIds: ['ws-1'], adminEnabled: true } }) + mockIsPlatformAdmin.mockResolvedValue(true) + expect( + await isFeatureEnabled('knowledge-member-access', { workspaceId: 'ws-2', userId: 'admin' }) + ).toBe(true) + mockIsPlatformAdmin.mockResolvedValue(false) + expect( + await isFeatureEnabled('knowledge-member-access', { workspaceId: 'ws-2', userId: 'u1' }) + ).toBe(false) + expect(await isFeatureEnabled('knowledge-member-access', { workspaceId: 'ws-2' })).toBe(false) + }) }) describe('credential-groups flag', () => { diff --git a/apps/sim/lib/core/config/feature-flags.ts b/apps/sim/lib/core/config/feature-flags.ts index e0d457d69c9..bf6de543ba9 100644 --- a/apps/sim/lib/core/config/feature-flags.ts +++ b/apps/sim/lib/core/config/feature-flags.ts @@ -28,16 +28,6 @@ export type FeatureFlagsConfig = Record */ export type FeatureFlagContext = AppConfigGateContext -/** - * Registry of known feature flags. Each maps to the secret consulted ONLY when - * AppConfig is not the source of truth (self-hosted/OSS, local dev, or hosted - * without APPCONFIG_*). A truthy secret turns the flag on globally. - * - * Gating by workspace/org/user/admin is available ONLY through the hosted AppConfig document - * — it deliberately cannot be expressed here, so no environment can grant (e.g.) - * admin access from a code literal. To add a flag, register its name and the secret - * to fall back on. - */ /** * The single definition of a feature flag. Everything about a flag lives in one * place: its name (the registry key), a human-readable `description`, and the @@ -85,6 +75,17 @@ const FEATURE_FLAGS = { 'Enterprise subscription. Off-AppConfig falls back to CREDENTIAL_GROUPS.', fallback: 'CREDENTIAL_GROUPS', }, + 'knowledge-member-access': { + description: + 'Permission-aware knowledge bases: lets a workspace admin sync a connector once per ' + + 'Credential Group member so each person sees only what their own account can read, and ' + + 'makes hybrid retrieval with a source-recency boost the default for searches in that ' + + 'workspace. Gated by workspaceId via AppConfig for members mode, which is judged by the ' + + 'workspace alone; the adminEnabled clause additionally opens the retrieval default to a ' + + 'platform admin anywhere. Off-AppConfig falls back to KNOWLEDGE_MEMBER_ACCESS. Requires ' + + 'the credential-groups flag for the connector side to do anything.', + fallback: 'KNOWLEDGE_MEMBER_ACCESS', + }, } satisfies Record /** diff --git a/apps/sim/lib/core/telemetry.ts b/apps/sim/lib/core/telemetry.ts index 3d1ca1b39a7..e5aa1c8a965 100644 --- a/apps/sim/lib/core/telemetry.ts +++ b/apps/sim/lib/core/telemetry.ts @@ -684,11 +684,14 @@ export const PlatformEvents = { knowledgeBaseId: string resultsCount: number workspaceId?: string + /** Whether the search ran as a person (`user`) or as the workspace (`workspace`). */ + accessScopeKind?: 'user' | 'workspace' }) => { trackPlatformEvent('platform.knowledge_base.searched', { 'knowledge_base.id': attrs.knowledgeBaseId, 'search.results_count': attrs.resultsCount, ...(attrs.workspaceId && { 'workspace.id': attrs.workspaceId }), + ...(attrs.accessScopeKind && { 'search.access_scope_kind': attrs.accessScopeKind }), }) }, diff --git a/apps/sim/lib/core/utils/browser-storage.ts b/apps/sim/lib/core/utils/browser-storage.ts index d6f4c131bd2..9ae44e9def9 100644 --- a/apps/sim/lib/core/utils/browser-storage.ts +++ b/apps/sim/lib/core/utils/browser-storage.ts @@ -4,7 +4,10 @@ */ import { createLogger } from '@sim/logger' -import type { FileAttachmentForApi } from '@/app/workspace/[workspaceId]/home/types' +import type { + ChatRequestMode, + FileAttachmentForApi, +} from '@/app/workspace/[workspaceId]/home/types' import type { ChatContext } from '@/stores/panel' const logger = createLogger('BrowserStorage') @@ -317,6 +320,8 @@ export interface MothershipHandoff { * chat and billing a second turn. */ resumeUserMessageId?: string + /** The request mode the withdrawn send asked for, so a retry stays the same kind of turn. */ + requestMode?: ChatRequestMode } interface StoredHandoff extends MothershipHandoff { @@ -365,6 +370,7 @@ export class MothershipHandoffStorage { : [...MothershipHandoffStorage.pendingContexts(workspaceId), ...contexts], ...(handoff.fileAttachments?.length ? { fileAttachments: handoff.fileAttachments } : {}), ...(handoff.resumeUserMessageId ? { resumeUserMessageId: handoff.resumeUserMessageId } : {}), + ...(handoff.requestMode ? { requestMode: handoff.requestMode } : {}), workspaceId, timestamp: Date.now(), }) @@ -424,6 +430,7 @@ export class MothershipHandoffStorage { return { ...(data.message ? { message: data.message } : {}), contexts, + ...(data.requestMode === 'ask' ? { requestMode: 'ask' as const } : {}), ...(Array.isArray(data.fileAttachments) && data.fileAttachments.length > 0 ? { fileAttachments: data.fileAttachments } : {}), diff --git a/apps/sim/lib/core/utils/with-route-handler.ts b/apps/sim/lib/core/utils/with-route-handler.ts index 3e225c3b9d4..eb6c91d1c96 100644 --- a/apps/sim/lib/core/utils/with-route-handler.ts +++ b/apps/sim/lib/core/utils/with-route-handler.ts @@ -1,5 +1,5 @@ import { createLogger, runWithRequestContext } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' +import { describeError, findCause, getErrorMessage, redactBoundParameters } from '@sim/utils/errors' import type { NextRequest } from 'next/server' import { NextResponse } from 'next/server' import { getRateLimitHeaders } from '@/lib/api/server/rate-limit-context' @@ -87,6 +87,29 @@ function traceIdFromTraceparent(header: string | null | undefined): string | und return match[1] } +/** + * What a wrapped error hides: a query failure from the database client carries + * the driver's reason and the Postgres code on its cause, and only the outer + * message names the query. The shared describer reads the deepest cause and + * strips bound parameter values, so user data never reaches the log. + */ +function errorDetail(error: unknown): { cause?: string; code?: string; causeStack?: string } { + if (!(error instanceof Error) || error.cause === undefined) return {} + const described = describeError(error) + /** Outside production the deepest cause's stack says where a wrapped failure was raised. */ + const deepest = findCause( + error, + (candidate): candidate is Error => candidate instanceof Error && candidate.cause === undefined + ) + return { + cause: `${described.name}: ${described.message}`, + ...(described.code ? { code: described.code } : {}), + ...(process.env.NODE_ENV !== 'production' && deepest?.stack + ? { causeStack: redactBoundParameters(deepest.stack) } + : {}), + } +} + /** * Wraps a Next.js API route handler with centralized error reporting. * @@ -118,7 +141,9 @@ export function withRouteHandler( response = await withPermissionGroupScope(() => handler(request, context)) } catch (error) { const duration = Date.now() - startTime - const message = getErrorMessage(error, 'Unknown error') + /** A query failure names its bound values in the message; they are user data. */ + const message = redactBoundParameters(getErrorMessage(error, 'Unknown error')) + const detail = errorDetail(error) if (request.signal.aborted) { logger.info('Client closed request', { duration, status: 499 }) response = options.clientAbortResponse @@ -132,7 +157,12 @@ export function withRouteHandler( if (typedError) { const typedStatus = typedError.statusCode if (typedStatus >= 500) { - logger.error('Unhandled route error', { duration, status: typedStatus, error: message }) + logger.error('Unhandled route error', { + duration, + status: typedStatus, + error: message, + ...detail, + }) } else { logger.warn('Typed route error', { duration, status: typedStatus, error: message }) } @@ -144,13 +174,13 @@ export function withRouteHandler( } if (options.unhandledErrorResponse) { - logger.error('Unhandled route error', { duration, error: message }) + logger.error('Unhandled route error', { duration, error: message, ...detail }) response = options.unhandledErrorResponse({ error, requestId }) applyResponseHeaders(response, request, requestId) return response } - logger.error('Unhandled route error', { duration, error: message }) + logger.error('Unhandled route error', { duration, error: message, ...detail }) response = NextResponse.json({ error: 'Internal server error', requestId }, { status: 500 }) applyResponseHeaders(response, request, requestId) return response diff --git a/apps/sim/lib/credential-groups/application/authorization.test.ts b/apps/sim/lib/credential-groups/application/authorization.test.ts index fec8b1a7bbb..5522e1402b9 100644 --- a/apps/sim/lib/credential-groups/application/authorization.test.ts +++ b/apps/sim/lib/credential-groups/application/authorization.test.ts @@ -2,18 +2,30 @@ * @vitest-environment node */ -import type { WorkflowExecutionDelegatedPrincipal } from '@sim/auth/principal' +import type { DelegatedPrincipal, WorkflowExecutionDelegatedPrincipal } from '@sim/auth/principal' import { beforeEach, describe, expect, it, vi } from 'vitest' import { compileCredentialGroupWorkflowAccessPolicy } from '@/lib/credential-groups/application/workflow-access-policy' import { credentialOperations } from '@/lib/credentials/application/operations' const mocks = vi.hoisted(() => ({ loadEnrollmentAccess: vi.fn(), + loadBinding: vi.fn(), requirePolicy: vi.fn(), })) vi.mock('@/lib/credential-groups/credentials', () => ({ loadCredentialGroupEnrollmentAccessForSubject: mocks.loadEnrollmentAccess, + loadManagedCredentialGroupBinding: mocks.loadBinding, + isManagedCredentialGroupBindingLive: (binding: { + managedOauthStatus: string + enrollmentStatus: string + groupStatus: string + optionStatus: string | null + }) => + binding.managedOauthStatus === 'active' && + ['in_progress', 'completed'].includes(binding.enrollmentStatus) && + binding.groupStatus === 'active' && + binding.optionStatus === 'active', })) vi.mock('@/lib/resource-policies/repository', () => ({ @@ -29,10 +41,23 @@ const context = { workspaceId: 'workspace-1', workspaceOrganizationId: null, allowPersonalApiKeys: true, + credentialId: 'credential-1', credentialGroupId: 'group-1', credentialGroupEnrollmentId: 'enrollment-1', } +const liveBinding = { + credentialId: 'credential-1', + workspaceId: 'workspace-1', + providerId: 'google-email', + credentialGroupId: 'group-1', + credentialGroupOptionId: 'option-1', + managedOauthStatus: 'active', + enrollmentStatus: 'completed', + groupStatus: 'active', + optionStatus: 'active', +} + function storedPolicy(allowedWorkflowIds: string[] = []) { return { id: 'policy-1', @@ -82,10 +107,21 @@ function executorPrincipal(): WorkflowExecutionDelegatedPrincipal { } } -function requireAccess( - principal: WorkflowExecutionDelegatedPrincipal, - accessContext = context -): Promise { +function copilotPrincipal(subjectUserId: string | null = 'user-1'): DelegatedPrincipal { + return { + kind: 'delegated', + serviceId: 'copilot', + ...(subjectUserId ? { subjectUserId } : {}), + workspaceId: 'workspace-1', + delegationId: 'copilot-tool:call-1', + audience: 'sim:managed-oauth-credentials', + issuedAt: new Date(Date.now() - 1_000), + expiresAt: new Date(Date.now() + 60_000), + resourceScope: { credentialId: 'credential-1', chatId: 'chat-1' }, + } +} + +function requireAccess(principal: DelegatedPrincipal, accessContext = context): Promise { return requireCredentialGroupCredentialAccess( principal, accessContext, @@ -101,6 +137,54 @@ describe('requireCredentialGroupCredentialAccess', () => { enrollmentId: 'enrollment-1', email: 'person@example.com', }) + mocks.loadBinding.mockResolvedValue(liveBinding) + }) + + it('denies a Chat turn once the credential group or its option is disabled', async () => { + mocks.loadBinding.mockResolvedValue({ ...liveBinding, optionStatus: 'disabled' }) + await expect(requireAccess(copilotPrincipal())).rejects.toMatchObject({ code: 'forbidden' }) + + mocks.loadBinding.mockResolvedValue({ ...liveBinding, groupStatus: 'disabled' }) + await expect(requireAccess(copilotPrincipal())).rejects.toMatchObject({ code: 'forbidden' }) + }) + + it('denies a workflow run the same way once the group or option is disabled', async () => { + mocks.loadBinding.mockResolvedValue({ ...liveBinding, optionStatus: 'disabled' }) + await expect(requireAccess(executorPrincipal())).rejects.toMatchObject({ code: 'forbidden' }) + expect(mocks.requirePolicy).not.toHaveBeenCalled() + }) + + it('denies a Chat turn for a credential with no OAuth binding, which a workflow may still hold', async () => { + mocks.loadBinding.mockResolvedValue(null) + await expect(requireAccess(copilotPrincipal())).rejects.toMatchObject({ code: 'forbidden' }) + await expect(requireAccess(executorPrincipal())).resolves.toBeUndefined() + }) + + it("allows a Chat turn to use only the credential under the signed-in user's own enrollment", async () => { + await expect(requireAccess(copilotPrincipal())).resolves.toBeUndefined() + expect(mocks.loadEnrollmentAccess).toHaveBeenCalledWith('group-1', { + kind: 'sim_user', + userId: 'user-1', + }) + + await expect( + requireAccess(copilotPrincipal(), { ...context, credentialGroupEnrollmentId: 'enrollment-2' }) + ).rejects.toMatchObject({ code: 'forbidden' }) + }) + + it('denies a Chat turn whose user holds no live enrollment, even for an allowlisted workflow', async () => { + mocks.requirePolicy.mockResolvedValue(storedPolicy(['workflow-1'])) + mocks.loadEnrollmentAccess.mockResolvedValue(null) + + await expect(requireAccess(copilotPrincipal())).rejects.toMatchObject({ code: 'forbidden' }) + }) + + it('denies a Chat turn with no Sim user subject before reading anything', async () => { + await expect(requireAccess(copilotPrincipal(null))).rejects.toMatchObject({ + code: 'forbidden', + }) + expect(mocks.requirePolicy).not.toHaveBeenCalled() + expect(mocks.loadEnrollmentAccess).not.toHaveBeenCalled() }) it('allows an external actor to use only their own enrollment', async () => { diff --git a/apps/sim/lib/credential-groups/application/authorization.ts b/apps/sim/lib/credential-groups/application/authorization.ts index 13cc444e606..10a4bda02a7 100644 --- a/apps/sim/lib/credential-groups/application/authorization.ts +++ b/apps/sim/lib/credential-groups/application/authorization.ts @@ -12,10 +12,18 @@ import type { import { OrchestrationError } from '@/lib/core/orchestration/types' import { credentialGroupWorkflowAccessPolicyCodec, + evaluateCredentialGroupActorCredentialAccess, evaluateCredentialGroupWorkflowAccess, } from '@/lib/credential-groups/application/workflow-access-policy' -import type { CredentialGroupCredentialListContext } from '@/lib/credential-groups/credentials' -import { loadCredentialGroupEnrollmentAccessForSubject } from '@/lib/credential-groups/credentials' +import type { + CredentialGroupCredentialListContext, + ManagedCredentialGroupBinding, +} from '@/lib/credential-groups/credentials' +import { + isManagedCredentialGroupBindingLive, + loadCredentialGroupEnrollmentAccessForSubject, + loadManagedCredentialGroupBinding, +} from '@/lib/credential-groups/credentials' import type { ResourcePolicyBindingFor } from '@/lib/resource-policies/registry' import { requireResourcePolicy } from '@/lib/resource-policies/repository' @@ -83,11 +91,72 @@ export function requireCredentialGroupWorkflowActor(principal: Principal): Princ return requireConsistentWorkflowSubject(principal, requireWorkflowExecutionPrincipal(principal)) } +/** + * Authorizes a person using their own Credential Group credential from Chat. + * The copilot delegation names the signed-in user and no workflow, so only the + * actor statement is evaluated: the credential must be the one collected under + * that user's own live enrollment. Nothing the model passes can widen this; + * the acting user is the delegation's subject, not a tool argument. + */ +async function requireCredentialGroupActorCredentialAccess( + principal: Extract, + context: CredentialGroupAuthorizationContext & { credentialGroupEnrollmentId: string }, + binding: ManagedCredentialGroupBinding | null, + resourcePolicy: ResourcePolicyBindingFor<'credential_group'> +): Promise { + const subject = resolvePrincipalSubject(principal) + if (subject?.kind !== 'sim_user' || !subject.userId) { + throw new OrchestrationError('forbidden', 'Credential Group actor access required') + } + /** Chat mints OAuth credentials only; a credential with no OAuth binding is not its to use. */ + if (!binding) { + throw new OrchestrationError('forbidden', 'Credential Group credential access denied') + } + const [policy, actorAccess] = await Promise.all([ + requireResourcePolicy({ + workspaceId: context.workspaceId, + resourceType: 'credential_group', + resourceId: context.credentialGroupId, + codec: credentialGroupWorkflowAccessPolicyCodec, + }), + loadCredentialGroupEnrollmentAccessForSubject(context.credentialGroupId, subject), + ]) + if (!actorAccess) { + throw new OrchestrationError('forbidden', 'Credential Group credential access denied') + } + const decision = evaluateCredentialGroupActorCredentialAccess({ + document: policy.document, + credentialGroupId: context.credentialGroupId, + selectedEnrollmentId: context.credentialGroupEnrollmentId, + actorEnrollmentId: actorAccess.enrollmentId, + resourcePolicy, + }) + if (decision.decision !== 'allow') { + throw new OrchestrationError('forbidden', 'Credential Group credential access denied') + } +} + export async function requireCredentialGroupCredentialAccess( principal: Principal, - context: CredentialGroupAuthorizationContext & { credentialGroupEnrollmentId: string }, + context: CredentialGroupAuthorizationContext & { + credentialId: string + credentialGroupEnrollmentId: string + }, resourcePolicy: ResourcePolicyBindingFor<'credential_group'> ): Promise { + /** + * A managed OAuth credential is usable only while its credential, enrollment, + * option, and group are all live, whoever is using it: an admin disabling the + * group or option denies the next mint from a workflow and from Chat alike. A + * managed MCP credential has no OAuth binding row and keeps its own checks. + */ + const binding = await loadManagedCredentialGroupBinding(context.credentialId) + if (binding && !isManagedCredentialGroupBindingLive(binding)) { + throw new OrchestrationError('forbidden', 'Credential Group credential access denied') + } + if (principal.kind === 'delegated' && principal.serviceId === 'copilot') { + return requireCredentialGroupActorCredentialAccess(principal, context, binding, resourcePolicy) + } const executionPrincipal = requireWorkflowExecutionPrincipal(principal) const currentWorkflow = requireCurrentWorkflow(principal) const subject = requireConsistentWorkflowSubject(principal, executionPrincipal) diff --git a/apps/sim/lib/credential-groups/application/manage-access.ts b/apps/sim/lib/credential-groups/application/manage-access.ts index cdb3b5e03f1..67082e7a791 100644 --- a/apps/sim/lib/credential-groups/application/manage-access.ts +++ b/apps/sim/lib/credential-groups/application/manage-access.ts @@ -12,6 +12,7 @@ import { credentialGroupOperations } from '@/lib/credential-groups/application/o import { compileCredentialGroupWorkflowAccessPolicy, credentialGroupWorkflowAccessPolicyCodec, + decodeCredentialGroupKnowledgeConnectorAccess, decodeCredentialGroupWorkflowAccessPolicy, } from '@/lib/credential-groups/application/workflow-access-policy' import { @@ -147,9 +148,17 @@ export const updateCredentialGroupAccess = defineAuthorizedWorkspaceUseCase({ throw new OrchestrationError('conflict', new ResourcePolicyRevisionConflictError().message) } decodeCredentialGroupWorkflowAccessPolicy(existingPolicy.document, context.credentialGroupId) + /** + * Knowledge connector grants are owned by each connector's own settings, so + * a workflow-access edit carries them forward untouched. + */ const document = compileCredentialGroupWorkflowAccessPolicy({ credentialGroupId: context.credentialGroupId, allowedWorkflowIds: input.allowedWorkflowIds, + knowledgeConnectorAccess: decodeCredentialGroupKnowledgeConnectorAccess( + existingPolicy.document, + context.credentialGroupId + ), }) if (input.allowedWorkflowIds.length > 0) { const workflows = await loadCredentialGroupWorkflowCatalog(context.workspaceId) diff --git a/apps/sim/lib/credential-groups/application/workflow-access-policy.test.ts b/apps/sim/lib/credential-groups/application/workflow-access-policy.test.ts index 151608c4f6c..4d2b5b08866 100644 --- a/apps/sim/lib/credential-groups/application/workflow-access-policy.test.ts +++ b/apps/sim/lib/credential-groups/application/workflow-access-policy.test.ts @@ -6,11 +6,17 @@ import { describe, expect, it } from 'vitest' import { compileCredentialGroupWorkflowAccessPolicy, credentialGroupWorkflowAccessPolicyCodec, + decodeCredentialGroupKnowledgeConnectorAccess, decodeCredentialGroupWorkflowAccessPolicy, + evaluateCredentialGroupActorCredentialAccess, + evaluateCredentialGroupKnowledgeConnectorAccess, evaluateCredentialGroupWorkflowAccess, requireDefaultCredentialGroupWorkflowAccessPolicy, } from '@/lib/credential-groups/application/workflow-access-policy' -import { CREDENTIAL_GROUP_WORKFLOW_ACCESS_LIMIT } from '@/lib/credential-groups/limits' +import { + CREDENTIAL_GROUP_KNOWLEDGE_CONNECTOR_ACCESS_LIMIT, + CREDENTIAL_GROUP_WORKFLOW_ACCESS_LIMIT, +} from '@/lib/credential-groups/limits' import type { ResourcePolicyBindingFor } from '@/lib/resource-policies/registry' const GROUP_ID = 'group-1' @@ -26,6 +32,17 @@ function policy(workflowIds: string[]) { }) } +function connectorPolicy( + knowledgeConnectorAccess: Array<{ credentialGroupOptionId: string; connectorIds: string[] }>, + workflowIds: string[] = [] +) { + return compileCredentialGroupWorkflowAccessPolicy({ + credentialGroupId: GROUP_ID, + allowedWorkflowIds: workflowIds, + knowledgeConnectorAccess, + }) +} + describe('Credential Group workflow access policy', () => { it('compiles actor ownership plus one deterministic deployment-only workflow statement', () => { expect(policy(['workflow-2', 'workflow-1'])).toEqual({ @@ -194,6 +211,206 @@ describe('Credential Group workflow access policy', () => { ).toThrow() }) + describe('knowledge connector access', () => { + it('compiles one sorted, option-conditioned statement per credential option after the workflow statement', () => { + const document = connectorPolicy( + [ + { credentialGroupOptionId: 'option-b', connectorIds: ['connector-2', 'connector-1'] }, + { credentialGroupOptionId: 'option-a', connectorIds: ['connector-3'] }, + { credentialGroupOptionId: 'option-empty', connectorIds: [] }, + ], + ['workflow-1'] + ) + expect(document.statements.map((statement) => statement.sid)).toEqual([ + 'CredentialGroupActorCredentialAccess', + 'WorkflowCredentialAccess', + 'KnowledgeConnectorCredentialAccess:option-a', + 'KnowledgeConnectorCredentialAccess:option-b', + ]) + expect(document.statements[3]).toEqual({ + sid: 'KnowledgeConnectorCredentialAccess:option-b', + effect: 'allow', + actions: ['credential_groups.credentials.use'], + principals: [ + { type: 'knowledge_connector', connectorId: 'connector-1' }, + { type: 'knowledge_connector', connectorId: 'connector-2' }, + ], + condition: { StringEquals: { 'credential_group:OptionId': 'option-b' } }, + }) + expect(decodeCredentialGroupKnowledgeConnectorAccess(document, GROUP_ID)).toEqual([ + { credentialGroupOptionId: 'option-a', connectorIds: ['connector-3'] }, + { credentialGroupOptionId: 'option-b', connectorIds: ['connector-1', 'connector-2'] }, + ]) + expect(decodeCredentialGroupWorkflowAccessPolicy(document, GROUP_ID)).toEqual(['workflow-1']) + }) + + it('rejects a connector bound to two options, repeats, and oversized options', () => { + expect(() => + connectorPolicy([ + { credentialGroupOptionId: 'option-a', connectorIds: ['connector-1'] }, + { credentialGroupOptionId: 'option-b', connectorIds: ['connector-1'] }, + ]) + ).toThrow('more than one credential option') + expect(() => + connectorPolicy([ + { credentialGroupOptionId: 'option-a', connectorIds: ['connector-1', 'connector-1'] }, + ]) + ).toThrow('repeats knowledge connector connector-1') + expect(() => + connectorPolicy([ + { credentialGroupOptionId: 'option-a', connectorIds: ['connector-1'] }, + { credentialGroupOptionId: 'option-a', connectorIds: ['connector-2'] }, + ]) + ).toThrow('repeats credential option option-a') + expect(() => + connectorPolicy([ + { + credentialGroupOptionId: 'option-a', + connectorIds: Array.from( + { length: CREDENTIAL_GROUP_KNOWLEDGE_CONNECTOR_ACCESS_LIMIT + 1 }, + (_, index) => `connector-${index}` + ), + }, + ]) + ).toThrow( + `more than ${CREDENTIAL_GROUP_KNOWLEDGE_CONNECTOR_ACCESS_LIMIT} knowledge connectors` + ) + }) + + it.each([ + [ + 'a connector statement before the workflow statement', + () => { + const document = connectorPolicy( + [{ credentialGroupOptionId: 'option-a', connectorIds: ['connector-1'] }], + ['workflow-1'] + ) + return { + statements: [document.statements[0], document.statements[2], document.statements[1]], + } + }, + ], + [ + 'unsorted option statements', + () => { + const document = connectorPolicy([ + { credentialGroupOptionId: 'option-a', connectorIds: ['connector-1'] }, + { credentialGroupOptionId: 'option-b', connectorIds: ['connector-2'] }, + ]) + return { + statements: [document.statements[0], document.statements[2], document.statements[1]], + } + }, + ], + [ + 'a SID naming a different option than its condition', + () => { + const document = connectorPolicy([ + { credentialGroupOptionId: 'option-a', connectorIds: ['connector-1'] }, + ]) + return { + statements: [ + document.statements[0], + { ...document.statements[1], sid: 'KnowledgeConnectorCredentialAccess:option-b' }, + ], + } + }, + ], + [ + 'a connector statement without an option condition', + () => { + const document = connectorPolicy([ + { credentialGroupOptionId: 'option-a', connectorIds: ['connector-1'] }, + ]) + return { + statements: [ + document.statements[0], + { ...document.statements[1], condition: undefined }, + ], + } + }, + ], + [ + 'a connector statement carrying a workflow principal', + () => { + const document = connectorPolicy([ + { credentialGroupOptionId: 'option-a', connectorIds: ['connector-1'] }, + ]) + return { + statements: [ + document.statements[0], + { + ...document.statements[1], + principals: [{ type: 'workflow', workflowId: 'workflow-1' }], + }, + ], + } + }, + ], + ])('rejects %s', (_name, replacement) => { + const candidate = { ...policy([]), ...replacement() } + expect(() => + credentialGroupWorkflowAccessPolicyCodec.parse(candidate, { + type: 'credential_group', + id: GROUP_ID, + }) + ).toThrow() + }) + + it('grants exactly the named connector for exactly the conditioned option', () => { + const document = connectorPolicy( + [{ credentialGroupOptionId: 'option-a', connectorIds: ['connector-1'] }], + ['workflow-1'] + ) + const decide = (connectorId: string, credentialGroupOptionId: string) => + evaluateCredentialGroupKnowledgeConnectorAccess({ + document, + credentialGroupId: GROUP_ID, + connectorId, + credentialGroupOptionId, + resourcePolicy: RESOURCE_POLICY, + }) + expect(decide('connector-1', 'option-a')).toEqual({ + decision: 'allow', + statementSid: 'KnowledgeConnectorCredentialAccess:option-a', + }) + expect(decide('connector-1', 'option-b')).toEqual({ decision: 'implicit_deny' }) + expect(decide('connector-2', 'option-a')).toEqual({ decision: 'implicit_deny' }) + }) + + it('never lets a connector statement satisfy an actor or workflow evaluation', () => { + const document = connectorPolicy([ + { credentialGroupOptionId: 'option-a', connectorIds: ['connector-1'] }, + ]) + expect( + evaluateCredentialGroupWorkflowAccess({ + document, + credentialGroupId: GROUP_ID, + selectedEnrollmentId: 'enrollment-2', + actorEnrollmentId: 'enrollment-1', + currentWorkflow: { + workflowId: 'workflow-1', + mode: 'deployment', + deploymentVersionId: 'version-1', + }, + resourcePolicy: RESOURCE_POLICY, + }) + ).toEqual({ decision: 'implicit_deny' }) + }) + + it('treats connector grants as a non-default policy', () => { + expect(() => + requireDefaultCredentialGroupWorkflowAccessPolicy({ + revision: 1, + document: connectorPolicy([ + { credentialGroupOptionId: 'option-a', connectorIds: ['connector-1'] }, + ]), + credentialGroupId: GROUP_ID, + }) + ).toThrow('non-default') + }) + }) + it('requires the trigger-created policy to be revision one with only actor access', () => { expect(() => requireDefaultCredentialGroupWorkflowAccessPolicy({ @@ -218,6 +435,21 @@ describe('Credential Group workflow access policy', () => { ).toThrow('non-default') }) + it("evaluates the actor statement alone when there is no workflow, granting only the actor's own credential", () => { + const document = policy(['workflow-1']) + const evaluate = (selectedEnrollmentId: string) => + evaluateCredentialGroupActorCredentialAccess({ + document, + credentialGroupId: GROUP_ID, + selectedEnrollmentId, + actorEnrollmentId: 'enrollment-1', + resourcePolicy: RESOURCE_POLICY, + }).decision + + expect(evaluate('enrollment-1')).toBe('allow') + expect(evaluate('enrollment-2')).toBe('implicit_deny') + }) + it('evaluates actor ownership and deployed workflow access through registered statements', () => { const document = policy(['workflow-1']) expect( diff --git a/apps/sim/lib/credential-groups/application/workflow-access-policy.ts b/apps/sim/lib/credential-groups/application/workflow-access-policy.ts index 29faac88e46..ff8a340ebac 100644 --- a/apps/sim/lib/credential-groups/application/workflow-access-policy.ts +++ b/apps/sim/lib/credential-groups/application/workflow-access-policy.ts @@ -1,7 +1,13 @@ import type { WorkflowExecutionAuthority } from '@sim/auth/principal' import { z } from 'zod' -import { CREDENTIAL_GROUP_WORKFLOW_ACCESS_LIMIT } from '@/lib/credential-groups/limits' -import { CREDENTIAL_GROUP_ACTOR_OWNS_CREDENTIAL_CONDITION_KEY } from '@/lib/resource-policies/conditions' +import { + CREDENTIAL_GROUP_KNOWLEDGE_CONNECTOR_ACCESS_LIMIT, + CREDENTIAL_GROUP_WORKFLOW_ACCESS_LIMIT, +} from '@/lib/credential-groups/limits' +import { + CREDENTIAL_GROUP_ACTOR_OWNS_CREDENTIAL_CONDITION_KEY, + CREDENTIAL_GROUP_OPTION_ID_CONDITION_KEY, +} from '@/lib/resource-policies/conditions' import { WORKFLOW_MODE_RESOURCE_POLICY_CONDITION_KEY } from '@/lib/resource-policies/conditions/workflow-mode' import { evaluateResourcePolicy, @@ -9,6 +15,7 @@ import { } from '@/lib/resource-policies/evaluator' import { credentialGroupActorResourcePolicyPrincipalSchema, + knowledgeConnectorResourcePolicyPrincipalSchema, workflowResourcePolicyPrincipalSchema, } from '@/lib/resource-policies/principals' import { @@ -19,6 +26,14 @@ import type { ResourcePolicyCodec } from '@/lib/resource-policies/types' export const CREDENTIAL_GROUP_WORKFLOW_ACCESS_SID = 'WorkflowCredentialAccess' export const CREDENTIAL_GROUP_ACTOR_ACCESS_SID = 'CredentialGroupActorCredentialAccess' +/** + * One statement per credential option, `KnowledgeConnectorCredentialAccess:`, + * naming the knowledge connectors that crawl with that option's credentials. The + * option lives in the SID so statements stay addressable, and in the condition so + * the evaluator enforces it. + */ +export const CREDENTIAL_GROUP_KNOWLEDGE_CONNECTOR_ACCESS_SID_PREFIX = + 'KnowledgeConnectorCredentialAccess:' export { CREDENTIAL_GROUP_CREDENTIAL_USE_ACTION } export const CREDENTIAL_GROUP_WORKFLOW_MODE_CONDITION_KEY = WORKFLOW_MODE_RESOURCE_POLICY_CONDITION_KEY @@ -87,6 +102,80 @@ const credentialGroupWorkflowAccessStatementSchema = z }) .strict() +function knowledgeConnectorAccessSid(credentialGroupOptionId: string): string { + return `${CREDENTIAL_GROUP_KNOWLEDGE_CONNECTOR_ACCESS_SID_PREFIX}${credentialGroupOptionId}` +} + +const credentialGroupKnowledgeConnectorAccessStatementSchema = z + .object({ + sid: z + .string() + .startsWith(CREDENTIAL_GROUP_KNOWLEDGE_CONNECTOR_ACCESS_SID_PREFIX) + .refine( + (sid) => + canonicalIdSchema.safeParse( + sid.slice(CREDENTIAL_GROUP_KNOWLEDGE_CONNECTOR_ACCESS_SID_PREFIX.length) + ).success, + 'Knowledge connector access SID must name a canonical credential option' + ), + effect: z.literal('allow'), + actions: z.tuple([z.literal(CREDENTIAL_GROUP_CREDENTIAL_USE_ACTION)]), + principals: z + .array(knowledgeConnectorResourcePolicyPrincipalSchema) + .min(1) + .max(CREDENTIAL_GROUP_KNOWLEDGE_CONNECTOR_ACCESS_LIMIT) + .superRefine((principals, ctx) => { + for (let index = 1; index < principals.length; index += 1) { + const previous = principals[index - 1].connectorId + const current = principals[index].connectorId + if (current === previous) { + ctx.addIssue({ + code: 'custom', + path: [index, 'connectorId'], + message: `Credential Group access repeats knowledge connector ${current}`, + }) + } else if (current < previous) { + ctx.addIssue({ + code: 'custom', + path: [index, 'connectorId'], + message: 'Credential Group knowledge connector access principals must be sorted', + }) + } + } + }), + condition: z + .object({ + StringEquals: z + .object({ + [CREDENTIAL_GROUP_OPTION_ID_CONDITION_KEY]: canonicalIdSchema, + }) + .strict(), + }) + .strict(), + }) + .strict() + .superRefine((statement, ctx) => { + const optionId = statement.condition.StringEquals[CREDENTIAL_GROUP_OPTION_ID_CONDITION_KEY] + if (statement.sid !== knowledgeConnectorAccessSid(optionId)) { + ctx.addIssue({ + code: 'custom', + path: ['sid'], + message: 'Knowledge connector access SID must name the option in its condition', + }) + } + }) + +const credentialGroupAccessStatementSchema = z.union([ + credentialGroupActorAccessStatementSchema, + credentialGroupWorkflowAccessStatementSchema, + credentialGroupKnowledgeConnectorAccessStatementSchema, +]) + +/** + * Statement order is fixed so the document has one canonical form: the actor + * statement, then at most one workflow statement, then knowledge connector + * statements sorted by option. Anything else is rejected rather than normalised. + */ export const credentialGroupWorkflowAccessPolicySchema = z .object({ version: z.literal(1), @@ -96,13 +185,50 @@ export const credentialGroupWorkflowAccessPolicySchema = z id: canonicalIdSchema, }) .strict(), - statements: z.union([ - z.tuple([credentialGroupActorAccessStatementSchema]), - z.tuple([ - credentialGroupActorAccessStatementSchema, - credentialGroupWorkflowAccessStatementSchema, - ]), - ]), + statements: z + .array(credentialGroupAccessStatementSchema) + .min(1) + .superRefine((statements, ctx) => { + if (statements[0].sid !== CREDENTIAL_GROUP_ACTOR_ACCESS_SID) { + ctx.addIssue({ + code: 'custom', + path: [0, 'sid'], + message: 'Credential Group access must begin with the actor statement', + }) + } + let index = 1 + if (statements[index]?.sid === CREDENTIAL_GROUP_WORKFLOW_ACCESS_SID) index += 1 + let previousOptionId: string | undefined + for (; index < statements.length; index += 1) { + const sid = statements[index].sid + if (!sid.startsWith(CREDENTIAL_GROUP_KNOWLEDGE_CONNECTOR_ACCESS_SID_PREFIX)) { + ctx.addIssue({ + code: 'custom', + path: [index, 'sid'], + message: + 'Credential Group access statements must be ordered actor, workflow, then knowledge connectors', + }) + continue + } + const optionId = sid.slice(CREDENTIAL_GROUP_KNOWLEDGE_CONNECTOR_ACCESS_SID_PREFIX.length) + if (previousOptionId !== undefined) { + if (optionId === previousOptionId) { + ctx.addIssue({ + code: 'custom', + path: [index, 'sid'], + message: `Credential Group access repeats credential option ${optionId}`, + }) + } else if (optionId < previousOptionId) { + ctx.addIssue({ + code: 'custom', + path: [index, 'sid'], + message: 'Credential Group knowledge connector access statements must be sorted', + }) + } + } + previousOptionId = optionId + } + }), }) .strict() @@ -110,6 +236,19 @@ export type CredentialGroupWorkflowAccessPolicy = z.output< typeof credentialGroupWorkflowAccessPolicySchema > +type CredentialGroupWorkflowAccessStatement = z.output< + typeof credentialGroupWorkflowAccessStatementSchema +> +type CredentialGroupKnowledgeConnectorAccessStatement = z.output< + typeof credentialGroupKnowledgeConnectorAccessStatementSchema +> + +/** The knowledge connectors one credential option backs. */ +export interface CredentialGroupKnowledgeConnectorAccess { + credentialGroupOptionId: string + connectorIds: string[] +} + export const credentialGroupWorkflowAccessPolicyCodec = { resourceType: 'credential_group', parse( @@ -124,6 +263,12 @@ export const credentialGroupWorkflowAccessPolicyCodec = { }, } as const satisfies ResourcePolicyCodec<'credential_group', CredentialGroupWorkflowAccessPolicy> +function requireCanonicalId(value: string, label: string): void { + if (!value.trim() || value !== value.trim() || value.length > 128) { + throw new Error(`Credential Group access ${label} must be canonical non-empty strings`) + } +} + function requireAllowedWorkflowIds(allowedWorkflowIds: readonly string[]): string[] { if (allowedWorkflowIds.length > CREDENTIAL_GROUP_WORKFLOW_ACCESS_LIMIT) { throw new Error( @@ -133,9 +278,7 @@ function requireAllowedWorkflowIds(allowedWorkflowIds: readonly string[]): strin const workflowIds = new Set() for (const workflowId of allowedWorkflowIds) { - if (!workflowId.trim() || workflowId !== workflowId.trim() || workflowId.length > 128) { - throw new Error('Credential Group access workflow IDs must be canonical non-empty strings') - } + requireCanonicalId(workflowId, 'workflow IDs') if (workflowIds.has(workflowId)) { throw new Error(`Credential Group access repeats workflow ${workflowId}`) } @@ -144,11 +287,61 @@ function requireAllowedWorkflowIds(allowedWorkflowIds: readonly string[]): strin return [...workflowIds].sort() } +/** + * Normalises knowledge connector access into its canonical form: options sorted, + * connectors sorted and unique, empty options dropped, and each connector bound + * to one option only — a connector crawls with exactly one credential slot. + */ +function requireKnowledgeConnectorAccess( + entries: readonly CredentialGroupKnowledgeConnectorAccess[] +): CredentialGroupKnowledgeConnectorAccess[] { + const byOption = new Map>() + const boundConnectors = new Set() + for (const entry of entries) { + requireCanonicalId(entry.credentialGroupOptionId, 'credential option IDs') + if (byOption.has(entry.credentialGroupOptionId)) { + throw new Error( + `Credential Group access repeats credential option ${entry.credentialGroupOptionId}` + ) + } + const connectorIds = new Set() + for (const connectorId of entry.connectorIds) { + requireCanonicalId(connectorId, 'knowledge connector IDs') + if (connectorIds.has(connectorId)) { + throw new Error(`Credential Group access repeats knowledge connector ${connectorId}`) + } + if (boundConnectors.has(connectorId)) { + throw new Error( + `Credential Group access binds knowledge connector ${connectorId} to more than one credential option` + ) + } + connectorIds.add(connectorId) + boundConnectors.add(connectorId) + } + if (connectorIds.size > CREDENTIAL_GROUP_KNOWLEDGE_CONNECTOR_ACCESS_LIMIT) { + throw new Error( + `Credential Group access cannot allow more than ${CREDENTIAL_GROUP_KNOWLEDGE_CONNECTOR_ACCESS_LIMIT} knowledge connectors per credential option` + ) + } + if (connectorIds.size > 0) byOption.set(entry.credentialGroupOptionId, connectorIds) + } + return [...byOption.entries()] + .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) + .map(([credentialGroupOptionId, connectorIds]) => ({ + credentialGroupOptionId, + connectorIds: [...connectorIds].sort(), + })) +} + export function compileCredentialGroupWorkflowAccessPolicy(input: { credentialGroupId: string allowedWorkflowIds: readonly string[] + knowledgeConnectorAccess?: readonly CredentialGroupKnowledgeConnectorAccess[] }): CredentialGroupWorkflowAccessPolicy { const allowedWorkflowIds = requireAllowedWorkflowIds(input.allowedWorkflowIds) + const knowledgeConnectorAccess = requireKnowledgeConnectorAccess( + input.knowledgeConnectorAccess ?? [] + ) const actorStatement = { sid: CREDENTIAL_GROUP_ACTOR_ACCESS_SID, effect: 'allow', @@ -160,46 +353,87 @@ export function compileCredentialGroupWorkflowAccessPolicy(input: { }, }, } as const + const workflowStatements = + allowedWorkflowIds.length === 0 + ? [] + : [ + { + sid: CREDENTIAL_GROUP_WORKFLOW_ACCESS_SID, + effect: 'allow', + actions: [CREDENTIAL_GROUP_CREDENTIAL_USE_ACTION], + principals: allowedWorkflowIds.map((workflowId) => ({ + type: 'workflow' as const, + workflowId, + })), + condition: { + StringEquals: { + [CREDENTIAL_GROUP_WORKFLOW_MODE_CONDITION_KEY]: 'deployment', + }, + }, + }, + ] + const knowledgeConnectorStatements = knowledgeConnectorAccess.map((entry) => ({ + sid: knowledgeConnectorAccessSid(entry.credentialGroupOptionId), + effect: 'allow', + actions: [CREDENTIAL_GROUP_CREDENTIAL_USE_ACTION], + principals: entry.connectorIds.map((connectorId) => ({ + type: 'knowledge_connector' as const, + connectorId, + })), + condition: { + StringEquals: { + [CREDENTIAL_GROUP_OPTION_ID_CONDITION_KEY]: entry.credentialGroupOptionId, + }, + }, + })) return credentialGroupWorkflowAccessPolicyCodec.parse( { version: 1, resource: { type: 'credential_group', id: input.credentialGroupId }, - statements: - allowedWorkflowIds.length === 0 - ? [actorStatement] - : [ - actorStatement, - { - sid: CREDENTIAL_GROUP_WORKFLOW_ACCESS_SID, - effect: 'allow', - actions: [CREDENTIAL_GROUP_CREDENTIAL_USE_ACTION], - principals: allowedWorkflowIds.map((workflowId) => ({ - type: 'workflow' as const, - workflowId, - })), - condition: { - StringEquals: { - [CREDENTIAL_GROUP_WORKFLOW_MODE_CONDITION_KEY]: 'deployment', - }, - }, - }, - ], + statements: [actorStatement, ...workflowStatements, ...knowledgeConnectorStatements], }, { type: 'credential_group', id: input.credentialGroupId } ) } -export function decodeCredentialGroupWorkflowAccessPolicy( +function parseCanonicalDocument( document: unknown, credentialGroupId: string -): string[] { - const canonical = credentialGroupWorkflowAccessPolicyCodec.parse(document, { +): CredentialGroupWorkflowAccessPolicy { + return credentialGroupWorkflowAccessPolicyCodec.parse(document, { type: 'credential_group', id: credentialGroupId, }) - return canonical.statements.length === 1 - ? [] - : canonical.statements[1].principals.map((principal) => principal.workflowId) +} + +export function decodeCredentialGroupWorkflowAccessPolicy( + document: unknown, + credentialGroupId: string +): string[] { + const canonical = parseCanonicalDocument(document, credentialGroupId) + const workflowStatement = canonical.statements.find( + (statement): statement is CredentialGroupWorkflowAccessStatement => + statement.sid === CREDENTIAL_GROUP_WORKFLOW_ACCESS_SID + ) + return workflowStatement + ? workflowStatement.principals.map((principal) => principal.workflowId) + : [] +} + +export function decodeCredentialGroupKnowledgeConnectorAccess( + document: unknown, + credentialGroupId: string +): CredentialGroupKnowledgeConnectorAccess[] { + const canonical = parseCanonicalDocument(document, credentialGroupId) + return canonical.statements + .filter((statement): statement is CredentialGroupKnowledgeConnectorAccessStatement => + statement.sid.startsWith(CREDENTIAL_GROUP_KNOWLEDGE_CONNECTOR_ACCESS_SID_PREFIX) + ) + .map((statement) => ({ + credentialGroupOptionId: + statement.condition.StringEquals[CREDENTIAL_GROUP_OPTION_ID_CONDITION_KEY], + connectorIds: statement.principals.map((principal) => principal.connectorId), + })) } export function requireDefaultCredentialGroupWorkflowAccessPolicy(input: { @@ -211,7 +445,15 @@ export function requireDefaultCredentialGroupWorkflowAccessPolicy(input: { input.document, input.credentialGroupId ) - if (input.revision !== 1 || allowedWorkflowIds.length !== 0) { + const knowledgeConnectorAccess = decodeCredentialGroupKnowledgeConnectorAccess( + input.document, + input.credentialGroupId + ) + if ( + input.revision !== 1 || + allowedWorkflowIds.length !== 0 || + knowledgeConnectorAccess.length !== 0 + ) { throw new Error('New resource was bound to a non-default resource policy') } } @@ -224,10 +466,7 @@ export function evaluateCredentialGroupWorkflowAccess(input: { currentWorkflow: WorkflowExecutionAuthority resourcePolicy: ResourcePolicyBindingFor<'credential_group'> }): ResourcePolicyDecision { - const document = credentialGroupWorkflowAccessPolicyCodec.parse(input.document, { - type: 'credential_group', - id: input.credentialGroupId, - }) + const document = parseCanonicalDocument(input.document, input.credentialGroupId) return evaluateResourcePolicy({ document, action: input.resourcePolicy.action, @@ -240,3 +479,51 @@ export function evaluateCredentialGroupWorkflowAccess(input: { }, }) } + +/** + * Decides whether the person acting on their own behalf, outside any workflow + * run, may use a credential: only the actor statement can match, and it grants + * exactly the credential collected under the actor's own enrollment. The + * workflow statements need a current workflow fact and never match here. + */ +export function evaluateCredentialGroupActorCredentialAccess(input: { + document: CredentialGroupWorkflowAccessPolicy + credentialGroupId: string + selectedEnrollmentId: string + actorEnrollmentId: string + resourcePolicy: ResourcePolicyBindingFor<'credential_group'> +}): ResourcePolicyDecision { + const document = parseCanonicalDocument(input.document, input.credentialGroupId) + return evaluateResourcePolicy({ + document, + action: input.resourcePolicy.action, + facts: { + credentialGroupActorEnrollmentId: input.actorEnrollmentId, + credentialGroupCredentialEnrollmentId: input.selectedEnrollmentId, + }, + }) +} + +/** + * Decides whether a knowledge connector may use a credential collected under + * one option. There is no actor and no workflow: the connector is the principal + * and the option is the only condition, so the actor and workflow statements can + * never match here. + */ +export function evaluateCredentialGroupKnowledgeConnectorAccess(input: { + document: CredentialGroupWorkflowAccessPolicy + credentialGroupId: string + connectorId: string + credentialGroupOptionId: string + resourcePolicy: ResourcePolicyBindingFor<'credential_group'> +}): ResourcePolicyDecision { + const document = parseCanonicalDocument(input.document, input.credentialGroupId) + return evaluateResourcePolicy({ + document, + action: input.resourcePolicy.action, + facts: { + currentKnowledgeConnector: { connectorId: input.connectorId }, + credentialGroupOptionId: input.credentialGroupOptionId, + }, + }) +} diff --git a/apps/sim/lib/credential-groups/credentials.ts b/apps/sim/lib/credential-groups/credentials.ts index 15d20929d5a..584e5fc7f80 100644 --- a/apps/sim/lib/credential-groups/credentials.ts +++ b/apps/sim/lib/credential-groups/credentials.ts @@ -7,14 +7,17 @@ import { credentialGroupEnrollment, user, } from '@sim/db/schema' -import { and, asc, eq, gt, inArray, or, sql } from 'drizzle-orm' +import { and, asc, eq, gt, inArray, or, type SQL, sql } from 'drizzle-orm' import { getCredentialGroupProviderId, isCredentialGroupProvider, } from '@/lib/credential-groups/providers' +import type { CredentialGroupEnrollmentStatus } from '@/lib/credential-groups/types' export const MAX_CREDENTIAL_GROUP_CREDENTIAL_PAGE_SIZE = 100 +export type ManagedOAuthCredentialStatus = 'active' | 'needs_reauth' | 'revoked' + export interface CredentialGroupCredentialListContext { credentialGroupId: string workspaceId: string @@ -32,6 +35,55 @@ export interface CredentialGroupCredentialReference { providerTenantId: string | null } +/** + * A credential collected under one option, in any state. Carries both statuses + * so a caller reconciling membership can tell a live credential from one that + * needs re-authorisation or whose enrollment was revoked. + */ +export interface CredentialGroupOptionCredentialReference + extends CredentialGroupCredentialReference { + managedOauthStatus: ManagedOAuthCredentialStatus + enrollmentStatus: CredentialGroupEnrollmentStatus +} + +/** Where a managed credential sits: its group and the option it was collected under. */ +export interface ManagedCredentialGroupBinding { + credentialId: string + workspaceId: string + providerId: string + credentialGroupId: string + credentialGroupOptionId: string + managedOauthStatus: ManagedOAuthCredentialStatus + enrollmentStatus: CredentialGroupEnrollmentStatus + groupStatus: 'active' | 'disabled' + /** Null when the option was removed from the group. */ + optionStatus: 'active' | 'disabled' | null +} + +/** Enrollment statuses under which a person's managed credentials count as theirs. */ +export const LIVE_ENROLLMENT_STATUSES = ['in_progress', 'completed'] as const + +/** + * Whether a managed credential may be used right now: the credential, its + * enrollment, its option, and its group are all live. Every consumer that + * mints a token from a binding checks this, so a disabled option or a revoked + * enrollment denies without waiting for a scope bump to invalidate the + * credential itself. + */ +export function isManagedCredentialGroupBindingLive( + binding: Pick< + ManagedCredentialGroupBinding, + 'managedOauthStatus' | 'enrollmentStatus' | 'groupStatus' | 'optionStatus' + > +): boolean { + return ( + binding.managedOauthStatus === 'active' && + (LIVE_ENROLLMENT_STATUSES as readonly string[]).includes(binding.enrollmentStatus) && + binding.groupStatus === 'active' && + binding.optionStatus === 'active' + ) +} + export interface CredentialGroupEnrollmentAccess { enrollmentId: string email: string @@ -54,6 +106,14 @@ interface ListCredentialGroupCredentialReferencesInput { credentialGroupOptionIds: string[] } +interface ListCredentialGroupOptionCredentialReferencesInput { + workspaceId: string + credentialGroupId: string + credentialGroupOptionId: string + limit: number + cursor?: string +} + /** Resolves a verified Sim user's active enrollment in one Credential Group. */ export async function loadCredentialGroupEnrollmentAccess( credentialGroupId: string, @@ -71,7 +131,7 @@ export async function loadCredentialGroupEnrollmentAccess( eq(user.id, userId), eq(user.emailVerified, true), eq(credentialGroupEnrollment.credentialGroupId, credentialGroupId), - inArray(credentialGroupEnrollment.status, ['in_progress', 'completed']) + inArray(credentialGroupEnrollment.status, [...LIVE_ENROLLMENT_STATUSES]) ) ) .limit(1) @@ -99,7 +159,7 @@ export async function loadCredentialGroupEnrollmentAccessForSubject( .where( and( eq(credentialGroupEnrollment.credentialGroupId, credentialGroupId), - inArray(credentialGroupEnrollment.status, ['in_progress', 'completed']), + inArray(credentialGroupEnrollment.status, [...LIVE_ENROLLMENT_STATUSES]), eq(credential.type, 'managed_oauth'), eq(credential.managedOauthStatus, 'active'), eq(credential.providerId, providerId), @@ -132,24 +192,75 @@ export async function loadCredentialGroupCredentialListContext( return row ?? null } -/** Lists one bounded page of active managed credentials without selecting token material. */ -export async function listCredentialGroupCredentialReferences({ - workspaceId, - credentialGroupId, - limit, - cursor, - email, - credentialProviderIds, - credentialGroupOptionIds, -}: ListCredentialGroupCredentialReferencesInput): Promise<{ - credentials: CredentialGroupCredentialReference[] - nextCursor: string | null -}> { - if (credentialGroupOptionIds.length === 0) { - if (cursor) throw new CredentialGroupCredentialCursorNotFoundError() - return { credentials: [], nextCursor: null } +/** Loads where a managed credential sits without selecting token material. */ +export async function loadManagedCredentialGroupBinding( + credentialId: string +): Promise { + const [row] = await db + .select({ + credentialId: credential.id, + workspaceId: credential.workspaceId, + providerId: credential.providerId, + credentialGroupId: credentialGroupEnrollment.credentialGroupId, + credentialGroupOptionId: credential.credentialGroupOptionId, + managedOauthStatus: credential.managedOauthStatus, + enrollmentStatus: credentialGroupEnrollment.status, + groupStatus: credentialGroup.status, + groupOptions: credentialGroup.options, + }) + .from(credential) + .innerJoin( + credentialGroupEnrollment, + eq(credentialGroupEnrollment.id, credential.credentialGroupEnrollmentId) + ) + .innerJoin(credentialGroup, eq(credentialGroup.id, credentialGroupEnrollment.credentialGroupId)) + .where(and(eq(credential.id, credentialId), eq(credential.type, 'managed_oauth'))) + .limit(1) + if (!row) return null + if (!row.providerId) throw new Error(`Managed credential ${row.credentialId} has no provider ID`) + if (!row.credentialGroupOptionId) { + throw new Error(`Managed credential ${row.credentialId} has no credential option`) + } + if (!row.managedOauthStatus) { + throw new Error(`Managed credential ${row.credentialId} has no managed OAuth status`) } + return { + credentialId: row.credentialId, + workspaceId: row.workspaceId, + providerId: row.providerId, + credentialGroupId: row.credentialGroupId, + credentialGroupOptionId: row.credentialGroupOptionId, + managedOauthStatus: row.managedOauthStatus, + enrollmentStatus: row.enrollmentStatus, + groupStatus: row.groupStatus, + optionStatus: + row.groupOptions.find((option) => option.id === row.credentialGroupOptionId)?.status ?? null, + } +} + +interface CredentialReferencePageRow { + id: string + email: string + displayName: string + providerId: string | null + providerSubjectId: string | null + providerTenantId: string | null + managedOauthStatus: ManagedOAuthCredentialStatus | null + enrollmentStatus: CredentialGroupEnrollmentStatus + createdAt: Date +} +/** + * One keyset page of managed credentials joined to their enrollment. The cursor + * is re-validated against the same conditions as the page, so a cursor that no + * longer satisfies the listing (the credential left the set) is refused rather + * than silently repositioned. + */ +async function pageCredentialReferences( + conditions: readonly (SQL | undefined)[], + limit: number, + cursor: string | undefined +): Promise<{ rows: CredentialReferencePageRow[]; nextCursor: string | null }> { let cursorPosition: { id: string; createdAt: Date } | undefined if (cursor) { const [cursorRow] = await db @@ -159,21 +270,7 @@ export async function listCredentialGroupCredentialReferences({ credentialGroupEnrollment, eq(credentialGroupEnrollment.id, credential.credentialGroupEnrollmentId) ) - .where( - and( - eq(credential.id, cursor), - eq(credential.workspaceId, workspaceId), - eq(credential.type, 'managed_oauth'), - eq(credential.managedOauthStatus, 'active'), - eq(credentialGroupEnrollment.credentialGroupId, credentialGroupId), - email ? eq(credentialGroupEnrollment.email, email) : undefined, - inArray(credential.credentialGroupOptionId, credentialGroupOptionIds), - credentialProviderIds?.length - ? inArray(credential.providerId, credentialProviderIds) - : undefined, - inArray(credentialGroupEnrollment.status, ['in_progress', 'completed']) - ) - ) + .where(and(eq(credential.id, cursor), ...conditions)) .limit(1) if (!cursorRow) throw new CredentialGroupCredentialCursorNotFoundError() cursorPosition = cursorRow @@ -187,6 +284,8 @@ export async function listCredentialGroupCredentialReferences({ providerId: credential.providerId, providerSubjectId: credential.providerSubjectId, providerTenantId: credential.providerTenantId, + managedOauthStatus: credential.managedOauthStatus, + enrollmentStatus: credentialGroupEnrollment.status, createdAt: credential.createdAt, }) .from(credential) @@ -196,16 +295,7 @@ export async function listCredentialGroupCredentialReferences({ ) .where( and( - eq(credential.workspaceId, workspaceId), - eq(credential.type, 'managed_oauth'), - eq(credential.managedOauthStatus, 'active'), - eq(credentialGroupEnrollment.credentialGroupId, credentialGroupId), - email ? eq(credentialGroupEnrollment.email, email) : undefined, - inArray(credential.credentialGroupOptionId, credentialGroupOptionIds), - credentialProviderIds?.length - ? inArray(credential.providerId, credentialProviderIds) - : undefined, - inArray(credentialGroupEnrollment.status, ['in_progress', 'completed']), + ...conditions, cursorPosition ? or( gt(credential.createdAt, cursorPosition.createdAt), @@ -224,21 +314,100 @@ export async function listCredentialGroupCredentialReferences({ const pageRows = hasMore ? rows.slice(0, limit) : rows const nextCursor = hasMore ? pageRows.at(-1)?.id : null if (hasMore && !nextCursor) throw new Error('Credential page cursor could not be derived') + return { rows: pageRows, nextCursor: nextCursor ?? null } +} + +function toCredentialReference( + row: CredentialReferencePageRow +): CredentialGroupCredentialReference { + if (!row.providerId) throw new Error(`Managed credential ${row.id} has no provider ID`) + if (!row.providerSubjectId) { + throw new Error(`Managed credential ${row.id} has no provider subject ID`) + } + return { + credentialId: row.id, + email: row.email, + displayName: row.displayName, + providerId: row.providerId, + providerSubjectId: row.providerSubjectId, + providerTenantId: row.providerTenantId, + } +} + +/** Lists one bounded page of active managed credentials without selecting token material. */ +export async function listCredentialGroupCredentialReferences({ + workspaceId, + credentialGroupId, + limit, + cursor, + email, + credentialProviderIds, + credentialGroupOptionIds, +}: ListCredentialGroupCredentialReferencesInput): Promise<{ + credentials: CredentialGroupCredentialReference[] + nextCursor: string | null +}> { + if (credentialGroupOptionIds.length === 0) { + if (cursor) throw new CredentialGroupCredentialCursorNotFoundError() + return { credentials: [], nextCursor: null } + } + + const page = await pageCredentialReferences( + [ + eq(credential.workspaceId, workspaceId), + eq(credential.type, 'managed_oauth'), + eq(credential.managedOauthStatus, 'active'), + eq(credentialGroupEnrollment.credentialGroupId, credentialGroupId), + email ? eq(credentialGroupEnrollment.email, email) : undefined, + inArray(credential.credentialGroupOptionId, credentialGroupOptionIds), + credentialProviderIds?.length + ? inArray(credential.providerId, credentialProviderIds) + : undefined, + inArray(credentialGroupEnrollment.status, [...LIVE_ENROLLMENT_STATUSES]), + ], + limit, + cursor + ) + return { credentials: page.rows.map(toCredentialReference), nextCursor: page.nextCursor } +} + +/** + * Lists one bounded page of every managed credential collected under one + * option, whatever its status. This is the reconciliation view: a caller that + * mirrors membership needs to see a credential that stopped being usable, not + * just the ones that still are. + */ +export async function listCredentialGroupOptionCredentialReferences({ + workspaceId, + credentialGroupId, + credentialGroupOptionId, + limit, + cursor, +}: ListCredentialGroupOptionCredentialReferencesInput): Promise<{ + credentials: CredentialGroupOptionCredentialReference[] + nextCursor: string | null +}> { + const page = await pageCredentialReferences( + [ + eq(credential.workspaceId, workspaceId), + eq(credential.type, 'managed_oauth'), + eq(credentialGroupEnrollment.credentialGroupId, credentialGroupId), + eq(credential.credentialGroupOptionId, credentialGroupOptionId), + ], + limit, + cursor + ) return { - credentials: pageRows.map((row) => { - if (!row.providerId) throw new Error(`Managed credential ${row.id} has no provider ID`) - if (!row.providerSubjectId) { - throw new Error(`Managed credential ${row.id} has no provider subject ID`) + credentials: page.rows.map((row) => { + if (!row.managedOauthStatus) { + throw new Error(`Managed credential ${row.id} has no managed OAuth status`) } return { - credentialId: row.id, - email: row.email, - displayName: row.displayName, - providerId: row.providerId, - providerSubjectId: row.providerSubjectId, - providerTenantId: row.providerTenantId, + ...toCredentialReference(row), + managedOauthStatus: row.managedOauthStatus, + enrollmentStatus: row.enrollmentStatus, } }), - nextCursor: nextCursor ?? null, + nextCursor: page.nextCursor, } } diff --git a/apps/sim/lib/credential-groups/enrollments.ts b/apps/sim/lib/credential-groups/enrollments.ts index a22ac3fd42a..7ac30a2c80a 100644 --- a/apps/sim/lib/credential-groups/enrollments.ts +++ b/apps/sim/lib/credential-groups/enrollments.ts @@ -64,9 +64,12 @@ interface InvitationContext { groupName: string } +/** What issuing an invitation does to an enrollment an admin revoked. */ +export type RevokedEnrollmentPolicy = 'reactivate' | 'reject' + interface SendInvitationOptions { expectedEnrollmentId?: string - revokedEnrollment: 'reactivate' | 'reject' + revokedEnrollment: RevokedEnrollmentPolicy } interface IssuedInvitation { @@ -772,11 +775,18 @@ export async function inviteCredentialGroupEnrollment( userId: string | undefined, /** See {@link sendInvitation}: absent for a workflow-issued invitation. */ inviterName: string | undefined, - email: string + email: string, + /** + * What a revoked enrollment does to the invitation, decided inside the + * issuing transaction. An admin's invite reactivates it; an automatic + * invitation rejects it, so a revocation that lands after the caller read + * the enrollment is never undone by a stale read. + */ + revokedEnrollment: RevokedEnrollmentPolicy = 'reactivate' ): Promise { const context = await getInvitationContext(workspaceId, groupId) return sendInvitation(context, userId, inviterName, normalizeEmail(email), { - revokedEnrollment: 'reactivate', + revokedEnrollment, }) } @@ -785,11 +795,13 @@ export async function createCredentialGroupInvitationLink( groupId: string, /** See {@link issueInvitation}: the issuer is attribution, never the authority. */ userId: string | undefined, - email: string + email: string, + /** See {@link inviteCredentialGroupEnrollment}. */ + revokedEnrollment: RevokedEnrollmentPolicy = 'reactivate' ): Promise { const context = await getInvitationContext(workspaceId, groupId) const issued = await issueInvitation(context, userId, normalizeEmail(email), { - revokedEnrollment: 'reactivate', + revokedEnrollment, }) return { enrollment: toCredentialGroupEnrollment(issued.enrollment), diff --git a/apps/sim/lib/credential-groups/limits.ts b/apps/sim/lib/credential-groups/limits.ts index 93e3a889f59..f7df344e79c 100644 --- a/apps/sim/lib/credential-groups/limits.ts +++ b/apps/sim/lib/credential-groups/limits.ts @@ -2,3 +2,5 @@ export const CREDENTIAL_GROUP_MCP_SERVER_LIMIT = 50 export const CREDENTIAL_GROUP_WORKFLOW_ACCESS_LIMIT = 50 export const CREDENTIAL_GROUP_WORKFLOW_CATALOG_LIMIT = 500 export const CREDENTIAL_GROUP_WORKFLOW_NAME_MAX_LENGTH = 255 +/** Knowledge connectors one credential option may back at once. */ +export const CREDENTIAL_GROUP_KNOWLEDGE_CONNECTOR_ACCESS_LIMIT = 32 diff --git a/apps/sim/lib/credential-groups/oauth.ts b/apps/sim/lib/credential-groups/oauth.ts index 107ba18a767..131df11b1c0 100644 --- a/apps/sim/lib/credential-groups/oauth.ts +++ b/apps/sim/lib/credential-groups/oauth.ts @@ -1,5 +1,7 @@ import { db } from '@sim/db' import { credential, credentialGroup, credentialGroupEnrollment } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { and, eq, ne, sql } from 'drizzle-orm' import { @@ -94,6 +96,8 @@ async function assertCurrentPolicy( return policy } +const logger = createLogger('CredentialGroupOAuth') + /** Builds a provider authorization URL after persisting a provider-bound one-time attempt. */ export async function startCredentialGroupOAuth( context: CredentialGroupOAuthContext, @@ -127,7 +131,7 @@ async function persistGrant( throw new CredentialGroupOAuthError('Provider returned a credential for another app.', 502) } - return db.transaction(async (tx) => { + const completion: CredentialGroupOAuthCompletion = await db.transaction(async (tx) => { await lockCredentialGroupEnrollmentLifecycle(tx, context.enrollmentId) await tx.execute( sql`SELECT pg_advisory_xact_lock(hashtextextended(${`credential-group-oauth:${context.enrollmentId}:${context.option.id}`}, 0))` @@ -293,6 +297,27 @@ async function persistGrant( enrollmentStatus, } }) + + /** + * Knowledge connectors crawling through this option pick the member up on + * their next run; queue one now so their documents arrive within minutes. + * Loaded lazily: credential groups do not otherwise depend on knowledge. + */ + try { + const { dispatchMemberSyncsForCredentialOption } = await import( + '@/lib/knowledge/connectors/member-queue' + ) + await dispatchMemberSyncsForCredentialOption({ + workspaceId: context.workspaceId, + credentialGroupOptionId: context.option.id, + }) + } catch (error) { + logger.warn('Failed to queue member syncs after an account connected', { + credentialGroupOptionId: context.option.id, + error: getErrorMessage(error), + }) + } + return completion } /** Exchanges a single-use code through its provider adapter and persists a normalized grant. */ diff --git a/apps/sim/lib/credential-groups/provider-registry.test.ts b/apps/sim/lib/credential-groups/provider-registry.test.ts index b9a3bb612c6..62a27c27d62 100644 --- a/apps/sim/lib/credential-groups/provider-registry.test.ts +++ b/apps/sim/lib/credential-groups/provider-registry.test.ts @@ -37,6 +37,18 @@ describe('Credential Group provider registry', () => { expect(getCredentialGroupProviderFromProviderId(service.providerId)).toBe('google-calendar') }) + it('maps Google Drive to its existing OAuth provider and the Google managed policy', () => { + const service = getCredentialGroupProviderService('google-drive') + + expect(service.name).toBe('Google Drive') + expect(service.providerId).toBe('google-drive') + expect(getCredentialGroupProviderFromProviderId(service.providerId)).toBe('google-drive') + expect(getCredentialGroupProviderAdapter('google-drive').provider).toBe('google-drive') + expect(getManagedOAuthConnectorPolicy('google-drive')?.getAuthorizationAppId('client')).toBe( + createGoogleManagedOAuthConnector('google-drive').getAuthorizationAppId('client') + ) + }) + it.each(['confluence', 'jira'] as const)( 'maps %s to its existing OAuth provider and adapter', (provider) => { diff --git a/apps/sim/lib/credential-groups/provider-registry.ts b/apps/sim/lib/credential-groups/provider-registry.ts index 08642b7e598..840bf6a145b 100644 --- a/apps/sim/lib/credential-groups/provider-registry.ts +++ b/apps/sim/lib/credential-groups/provider-registry.ts @@ -12,11 +12,23 @@ const CREDENTIAL_GROUP_PROVIDER_ADAPTERS: Record< > = { gmail: createStandardOAuthCredentialGroupProviderAdapter('gmail'), 'google-calendar': createStandardOAuthCredentialGroupProviderAdapter('google-calendar'), + 'google-drive': createStandardOAuthCredentialGroupProviderAdapter('google-drive'), + 'google-docs': createStandardOAuthCredentialGroupProviderAdapter('google-docs'), + 'google-forms': createStandardOAuthCredentialGroupProviderAdapter('google-forms'), + 'google-chat': createStandardOAuthCredentialGroupProviderAdapter('google-chat'), + 'google-meet': createStandardOAuthCredentialGroupProviderAdapter('google-meet'), + 'google-sheets': createStandardOAuthCredentialGroupProviderAdapter('google-sheets'), + 'microsoft-teams': createStandardOAuthCredentialGroupProviderAdapter('microsoft-teams'), + outlook: createStandardOAuthCredentialGroupProviderAdapter('outlook'), + onedrive: createStandardOAuthCredentialGroupProviderAdapter('onedrive'), + sharepoint: createStandardOAuthCredentialGroupProviderAdapter('sharepoint'), + 'microsoft-excel': createStandardOAuthCredentialGroupProviderAdapter('microsoft-excel'), confluence: createStandardOAuthCredentialGroupProviderAdapter('confluence'), jira: createStandardOAuthCredentialGroupProviderAdapter('jira'), airtable: createStandardOAuthCredentialGroupProviderAdapter('airtable'), asana: createStandardOAuthCredentialGroupProviderAdapter('asana'), attio: createStandardOAuthCredentialGroupProviderAdapter('attio'), + bitbucket: createStandardOAuthCredentialGroupProviderAdapter('bitbucket'), box: createStandardOAuthCredentialGroupProviderAdapter('box'), calcom: createStandardOAuthCredentialGroupProviderAdapter('calcom'), clickup: createStandardOAuthCredentialGroupProviderAdapter('clickup'), diff --git a/apps/sim/lib/credential-groups/providers.ts b/apps/sim/lib/credential-groups/providers.ts index 07cce8fe04d..09638d94ef2 100644 --- a/apps/sim/lib/credential-groups/providers.ts +++ b/apps/sim/lib/credential-groups/providers.ts @@ -4,11 +4,23 @@ import { getServiceConfigByServiceId } from '@/lib/oauth' export const CREDENTIAL_GROUP_STANDARD_OAUTH_PROVIDER_IDS = [ 'gmail', 'google-calendar', + 'google-drive', + 'google-docs', + 'google-forms', + 'google-chat', + 'google-meet', + 'google-sheets', + 'microsoft-teams', + 'outlook', + 'onedrive', + 'sharepoint', + 'microsoft-excel', 'confluence', 'jira', 'airtable', 'asana', 'attio', + 'bitbucket', 'box', 'calcom', 'clickup', @@ -55,6 +67,61 @@ const CREDENTIAL_GROUP_PROVIDER_SUPPORT: Record< description: 'Let each person connect one Google Calendar account', configuration: 'oauth', }, + 'google-drive': { + serviceId: 'google-drive', + description: 'Let each person connect one Google Drive account', + configuration: 'oauth', + }, + 'google-docs': { + serviceId: 'google-docs', + description: 'Let each person connect one Google Docs account', + configuration: 'oauth', + }, + 'google-forms': { + serviceId: 'google-forms', + description: 'Let each person connect one Google Forms account', + configuration: 'oauth', + }, + 'google-chat': { + serviceId: 'google-chat', + description: 'Let each person connect one Google Chat account', + configuration: 'oauth', + }, + 'google-meet': { + serviceId: 'google-meet', + description: 'Let each person connect one Google Meet account', + configuration: 'oauth', + }, + 'google-sheets': { + serviceId: 'google-sheets', + description: 'Let each person connect one Google Sheets account', + configuration: 'oauth', + }, + 'microsoft-teams': { + serviceId: 'microsoft-teams', + description: 'Let each person connect one Microsoft Teams account', + configuration: 'oauth', + }, + outlook: { + serviceId: 'outlook', + description: 'Let each person connect one Outlook account', + configuration: 'oauth', + }, + onedrive: { + serviceId: 'onedrive', + description: 'Let each person connect one OneDrive account', + configuration: 'oauth', + }, + sharepoint: { + serviceId: 'sharepoint', + description: 'Let each person connect one SharePoint account', + configuration: 'oauth', + }, + 'microsoft-excel': { + serviceId: 'microsoft-excel', + description: 'Let each person connect one Microsoft Excel account', + configuration: 'oauth', + }, confluence: { serviceId: 'confluence', description: 'Let each person connect one Confluence account', @@ -80,6 +147,11 @@ const CREDENTIAL_GROUP_PROVIDER_SUPPORT: Record< description: 'Let each person connect one Attio account', configuration: 'oauth', }, + bitbucket: { + serviceId: 'bitbucket', + description: 'Let each person connect one Bitbucket account', + configuration: 'oauth', + }, box: { serviceId: 'box', description: 'Let each person connect one Box account', diff --git a/apps/sim/lib/credential-groups/service.ts b/apps/sim/lib/credential-groups/service.ts index c754b11a6d4..590171903f7 100644 --- a/apps/sim/lib/credential-groups/service.ts +++ b/apps/sim/lib/credential-groups/service.ts @@ -4,10 +4,13 @@ import { credential, credentialGroup, credentialGroupEnrollment, + knowledgeBase, + knowledgeConnector, mcpServers, } from '@sim/db/schema' import { generateId } from '@sim/utils/id' import { and, asc, desc, eq, inArray, isNull } from 'drizzle-orm' +import { OrchestrationError } from '@/lib/core/orchestration/types' import { credentialGroupWorkflowAccessPolicyCodec, requireDefaultCredentialGroupWorkflowAccessPolicy, @@ -284,6 +287,53 @@ export async function createCredentialGroup( }) } +/** + * Refuses to remove a group, or the given options of it, while a knowledge + * connector syncs per member through one of them: the connector would be left + * bound to nothing, and its members' documents dark, without anyone choosing + * that. `optionIds` null means the whole group. + * + * Runs under the caller's `FOR UPDATE` on the group row. Every write that binds + * a connector row to an option (`lockCredentialGroupOption`) takes that same + * lock and re-checks the option under it, so a binding is either visible here + * or refused once this transaction commits; the check reads only the rows. + */ +async function refuseIfServingMemberConnectors( + executor: DbOrTx, + workspaceId: string, + groupId: string, + optionIds: readonly string[] | null +): Promise { + if (optionIds !== null && optionIds.length === 0) return + const serving = await executor + .select({ + knowledgeBaseName: knowledgeBase.name, + connectorType: knowledgeConnector.connectorType, + }) + .from(knowledgeConnector) + .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) + .where( + and( + eq(knowledgeBase.workspaceId, workspaceId), + eq(knowledgeConnector.accessMode, 'members'), + eq(knowledgeConnector.credentialGroupId, groupId), + optionIds === null + ? undefined + : inArray(knowledgeConnector.credentialGroupOptionId, [...optionIds]), + isNull(knowledgeConnector.deletedAt) + ) + ) + .limit(5) + if (serving.length === 0) return + const names = serving + .map((row) => `the ${row.connectorType} connector in "${row.knowledgeBaseName}"`) + .join(', ') + throw new OrchestrationError( + 'conflict', + `${optionIds === null ? 'This Credential Group' : 'An option being removed'} is what ${names} ${serving.length === 1 ? 'syncs' : 'sync'} per member through. Switch ${serving.length === 1 ? 'that connector' : 'those connectors'} to another group first.` + ) +} + export async function deleteCredentialGroup( workspaceId: string, groupId: string @@ -301,6 +351,7 @@ export async function deleteCredentialGroup( const retiredMcp = await retireManagedMcpServersForGroup(workspaceId, groupId, tx) + await refuseIfServingMemberConnectors(tx, workspaceId, groupId, null) await deleteResourcePolicyForResource( { workspaceId, resourceType: 'credential_group', resourceId: groupId }, tx @@ -332,6 +383,17 @@ export async function updateCredentialGroup( .for('update') if (!existing) return null + if (body.options !== undefined) { + const keptOptionIds = new Set(body.options.map((option) => option.id)) + await refuseIfServingMemberConnectors( + tx, + workspaceId, + groupId, + existing.options + .filter((option) => !keptOptionIds.has(option.id)) + .map((option) => option.id) + ) + } const nextOptions = body.options !== undefined ? await updateOptions(workspaceId, groupId, body.options, existing.options, tx) diff --git a/apps/sim/lib/credentials/application/copilot-managed-oauth-delegation.test.ts b/apps/sim/lib/credentials/application/copilot-managed-oauth-delegation.test.ts new file mode 100644 index 00000000000..372d0fbe909 --- /dev/null +++ b/apps/sim/lib/credentials/application/copilot-managed-oauth-delegation.test.ts @@ -0,0 +1,39 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { createCopilotManagedOAuthPrincipal } from '@/lib/credentials/application/copilot-managed-oauth-delegation' + +const trustedContext = { + userId: 'user-1', + workspaceId: 'workspace-1', + chatId: 'chat-1', + toolCallId: 'call-1', + copilotToolExecution: true as const, +} + +describe('createCopilotManagedOAuthPrincipal', () => { + it('names the signed-in user, the managed-credential audience, and exactly one credential', () => { + const principal = createCopilotManagedOAuthPrincipal(trustedContext, 'credential-1') + + expect(principal).toMatchObject({ + kind: 'delegated', + serviceId: 'copilot', + subjectUserId: 'user-1', + workspaceId: 'workspace-1', + delegationId: 'copilot-tool:call-1', + audience: 'sim:managed-oauth-credentials', + resourceScope: { credentialId: 'credential-1', chatId: 'chat-1' }, + }) + expect(principal.expiresAt.getTime()).toBeGreaterThan(principal.issuedAt.getTime()) + }) + + it('refuses a context the server did not classify as a Chat tool call', () => { + expect(() => + createCopilotManagedOAuthPrincipal({ ...trustedContext, copilotToolExecution: false }, 'c-1') + ).toThrow('trusted Copilot execution context') + expect(() => createCopilotManagedOAuthPrincipal(undefined, 'c-1')).toThrow( + 'Copilot execution context is required' + ) + }) +}) diff --git a/apps/sim/lib/credentials/application/copilot-managed-oauth-delegation.ts b/apps/sim/lib/credentials/application/copilot-managed-oauth-delegation.ts new file mode 100644 index 00000000000..4df285fca00 --- /dev/null +++ b/apps/sim/lib/credentials/application/copilot-managed-oauth-delegation.ts @@ -0,0 +1,28 @@ +import type { DelegatedPrincipal } from '@sim/auth/principal' +import { + COPILOT_APPLICATION_DELEGATION_TTL_MS, + type CopilotExecutionContext, + createCopilotApplicationPrincipal, + requireTrustedCopilotExecutionContext, +} from '@/lib/copilot/auth/application-delegation' +import { MANAGED_OAUTH_DELEGATION_AUDIENCE } from '@/lib/credentials/application/authorization' + +/** + * The principal a Chat tool call presents for one managed credential: a copilot + * delegation naming the signed-in user, scoped to that credential, with no + * workflow. The credential-group authorization evaluates its actor statement + * against this subject, so the person can use the credential they collected + * under their own enrollment and nothing else. + */ +export function createCopilotManagedOAuthPrincipal( + context: CopilotExecutionContext | undefined, + credentialId: string +): DelegatedPrincipal { + const trustedContext = requireTrustedCopilotExecutionContext(context) + return createCopilotApplicationPrincipal(trustedContext, { + audience: MANAGED_OAUTH_DELEGATION_AUDIENCE, + ttlMs: COPILOT_APPLICATION_DELEGATION_TTL_MS, + createDelegationId: (trusted) => `copilot-tool:${trusted.toolCallId}`, + resourceScope: { credentialId }, + }) +} diff --git a/apps/sim/lib/credentials/application/operations.ts b/apps/sim/lib/credentials/application/operations.ts index c851eaac563..df3e52834c6 100644 --- a/apps/sim/lib/credentials/application/operations.ts +++ b/apps/sim/lib/credentials/application/operations.ts @@ -177,7 +177,7 @@ export const credentialOperations = { workspaceApiKey: 'deny', capability: 'integrations.manage', principalKinds: ['delegated'], - delegatedServices: ['executor'], + delegatedServices: ['executor', 'copilot'], resourcePolicy: { resourceType: 'credential_group', action: CREDENTIAL_GROUP_CREDENTIAL_USE_ACTION, diff --git a/apps/sim/lib/credentials/environment.ts b/apps/sim/lib/credentials/environment.ts index 9756c4659e7..569f5663b22 100644 --- a/apps/sim/lib/credentials/environment.ts +++ b/apps/sim/lib/credentials/environment.ts @@ -1,8 +1,11 @@ import { db } from '@sim/db' import { credential, + credentialGroup, + credentialGroupEnrollment, credentialMember, permissions, + user, workspace, workspaceEnvironment, } from '@sim/db/schema' @@ -11,6 +14,7 @@ import { chunkArray } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' import { and, asc, eq, inArray, isNotNull, isNull, notInArray, or, sql } from 'drizzle-orm' import { acquireUserBillingIdentityLock } from '@/lib/billing/organizations/billing-identity-lock' +import { isManagedCredentialGroupBindingLive } from '@/lib/credential-groups/credentials' import type { DbOrTx } from '@/lib/db/types' import { getEffectiveWorkspacePermission, @@ -829,11 +833,80 @@ export interface AccessibleOAuthCredential { providerId: string displayName: string role: 'admin' | 'member' - /** Distinguishes a personal OAuth connection from a shared service account. */ - type: 'oauth' | 'service_account' + /** + * A personal OAuth connection, a shared service account, or a Credential + * Group credential the person collected under their own enrollment. + */ + type: 'oauth' | 'service_account' | 'managed_oauth' updatedAt: Date } +/** + * The Credential Group credentials a verified person holds through their own + * enrollments in the workspace and may use right now: the credential, its + * enrollment, its option, and its group are all live, the same bar every mint + * applies. These are theirs to use as themselves; the policy's actor statement + * is what a use is authorized against, so nothing here widens access, it only + * tells the person (and the agent acting for them) what exists. + */ +export async function getEnrolledManagedOAuthCredentials( + workspaceId: string, + userId: string +): Promise { + const rows = await db + .select({ + id: credential.id, + providerId: credential.providerId, + displayName: credential.displayName, + credentialGroupOptionId: credential.credentialGroupOptionId, + managedOauthStatus: credential.managedOauthStatus, + enrollmentStatus: credentialGroupEnrollment.status, + groupName: credentialGroup.name, + groupStatus: credentialGroup.status, + groupOptions: credentialGroup.options, + updatedAt: credential.updatedAt, + }) + .from(credential) + .innerJoin( + credentialGroupEnrollment, + eq(credentialGroupEnrollment.id, credential.credentialGroupEnrollmentId) + ) + .innerJoin(credentialGroup, eq(credentialGroup.id, credentialGroupEnrollment.credentialGroupId)) + .innerJoin(user, eq(sql`lower(btrim(${user.email}))`, credentialGroupEnrollment.email)) + .where( + and( + eq(credential.workspaceId, workspaceId), + eq(credentialGroup.workspaceId, workspaceId), + eq(credential.type, 'managed_oauth'), + eq(user.id, userId), + eq(user.emailVerified, true) + ) + ) + + return rows + .filter( + (row): row is typeof row & { providerId: string } => + Boolean(row.providerId) && + row.managedOauthStatus !== null && + isManagedCredentialGroupBindingLive({ + managedOauthStatus: row.managedOauthStatus, + enrollmentStatus: row.enrollmentStatus, + groupStatus: row.groupStatus, + optionStatus: + row.groupOptions.find((option) => option.id === row.credentialGroupOptionId)?.status ?? + null, + }) + ) + .map((row) => ({ + id: row.id, + providerId: row.providerId, + displayName: `${row.displayName} (${row.groupName})`, + role: 'member' as const, + type: 'managed_oauth' as const, + updatedAt: row.updatedAt, + })) +} + export async function getAccessibleOAuthCredentials( workspaceId: string, userId: string, diff --git a/apps/sim/lib/execution/payloads/materialization.server.ts b/apps/sim/lib/execution/payloads/materialization.server.ts index ec3f7ef2509..64ade013d81 100644 --- a/apps/sim/lib/execution/payloads/materialization.server.ts +++ b/apps/sim/lib/execution/payloads/materialization.server.ts @@ -17,6 +17,7 @@ import { MAX_INLINE_MATERIALIZATION_BYTES, } from '@/lib/execution/payloads/limits' import { ExecutionResourceLimitError } from '@/lib/execution/resource-errors' +import { resolveKnowledgeAccessScope } from '@/lib/knowledge/access/scope' import type { StorageContext } from '@/lib/uploads' import type { WorkspaceFileSecretProvenanceIdentity } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' import { @@ -313,7 +314,18 @@ export async function assertUserFileContentAccess( } const { verifyFileAccess } = await import('@/app/api/files/authorization') - const hasAccess = await verifyFileAccess(file.key, options.userId, undefined, context, false) + /** + * A knowledge-base file is read as the principal behind the run, when there + * is one; `options.userId` alone may be the workflow owner standing in for an + * actorless run and must not widen what the run can read. + */ + const knowledgeAccess = + context === 'knowledge-base' && options.principal + ? await resolveKnowledgeAccessScope(options.principal, { workspaceId: options.workspaceId }) + : undefined + const hasAccess = await verifyFileAccess(file.key, options.userId, undefined, context, false, { + knowledgeAccess, + }) if (!hasAccess) { throw new Error('File is not available in this execution.') } diff --git a/apps/sim/lib/integrations/credential-visibility.server.ts b/apps/sim/lib/integrations/credential-visibility.server.ts index 1837aa67571..929f18cfb7b 100644 --- a/apps/sim/lib/integrations/credential-visibility.server.ts +++ b/apps/sim/lib/integrations/credential-visibility.server.ts @@ -16,7 +16,7 @@ import { isHiddenUnder } from '@/blocks/visibility/context' export interface IntegrationCredentialIdentity { providerId: string - type?: 'oauth' | 'service_account' + type?: 'oauth' | 'service_account' | 'managed_oauth' } interface IntegrationCredentialVisibilityOptions { diff --git a/apps/sim/lib/knowledge/access/availability.ts b/apps/sim/lib/knowledge/access/availability.ts new file mode 100644 index 00000000000..6496b0e4b9d --- /dev/null +++ b/apps/sim/lib/knowledge/access/availability.ts @@ -0,0 +1,53 @@ +import { + getWorkspaceOwnerSubscriptionAccess, + type WorkspaceOwnerSubscriptionAccess, +} from '@/lib/billing/core/workspace-access' +import { isFeatureEnabled } from '@/lib/core/config/feature-flags' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { isCredentialGroupsAvailable } from '@/lib/credential-groups/availability' + +/** + * Who is asking. Members mode — creating, switching, syncing, and honouring + * member tokens — is judged by the workspace alone, because the member engine + * has no person to speak for and every gate must agree with it. Retrieval + * defaults pass the signed-in user as well, so the flag's platform-admin + * clause lets an admin try hybrid retrieval anywhere; an actorless caller + * (a schedule, a cron, a workspace API key) passes none. + */ +export interface KnowledgeMemberAccessContext { + workspaceId: string + userId?: string + /** The workspace owner's plan, when the caller already holds it. */ + ownerBilling?: WorkspaceOwnerSubscriptionAccess +} + +/** + * Whether permission-aware knowledge is on for this workspace: the + * `knowledge-member-access` flag, and Credential Groups available to the + * workspace, which members mode enrolls people through. Every gate the + * feature has checks this one function — creating and switching connectors, + * the member engine, the member tokens a reader is granted, and the + * workspace host context the UI reads — so they can never disagree. When it + * turns off, member-scoped documents are hidden on the next read, members-mode + * connectors wait rather than change anything, and search returns to the + * semantic-only default; nothing is deleted. + */ +export async function isKnowledgeMemberAccessAvailable( + context: KnowledgeMemberAccessContext +): Promise { + if (!(await isFeatureEnabled('knowledge-member-access', context))) return false + const ownerBilling = + context.ownerBilling ?? (await getWorkspaceOwnerSubscriptionAccess(context.workspaceId)) + return isCredentialGroupsAvailable({ workspaceId: context.workspaceId, ownerBilling }) +} + +/** Refuses with the one message every members-mode gate uses when the feature is off for the workspace. */ +export async function requireKnowledgeMemberAccessAvailable( + context: KnowledgeMemberAccessContext +): Promise { + if (await isKnowledgeMemberAccessAvailable(context)) return + throw new OrchestrationError( + 'validation', + 'Per-member access is not available for this workspace' + ) +} diff --git a/apps/sim/lib/knowledge/access/predicate.test.ts b/apps/sim/lib/knowledge/access/predicate.test.ts new file mode 100644 index 00000000000..3b7db9828f3 --- /dev/null +++ b/apps/sim/lib/knowledge/access/predicate.test.ts @@ -0,0 +1,58 @@ +/** + * @vitest-environment node + */ + +/** + * Renders the real predicate against the real drizzle dialect and schema. The + * shared client sets `fetch_types: false` (packages/db/db.ts), under which an + * array bound as one parameter fails at execution with 22P02, so the assertion + * that matters is that every bind is a scalar. + */ +import { describe, expect, it, vi } from 'vitest' + +vi.unmock('drizzle-orm') +vi.unmock('@sim/db') +vi.unmock('@sim/db/schema') + +process.env.DATABASE_URL ??= 'postgresql://user:pass@localhost:5432/test' + +const { PgDialect } = await import('drizzle-orm/pg-core') +const { knowledgeAccessCondition } = await import('@/lib/knowledge/access/predicate') +const { SYSTEM_ACCESS_SCOPE } = await import('@/lib/knowledge/access/types') + +function render(condition: ReturnType) { + return new PgDialect().sqlToQuery(condition) +} + +describe('knowledgeAccessCondition', () => { + it('overlaps the ACL with the tokens as a literal array of scalar binds', () => { + const { sql, params } = render( + knowledgeAccessCondition({ + kind: 'user', + userId: 'user-1', + tokens: ['pub', 's:confluence:-:557058:abc', 'ws'], + }) + ) + expect(sql).toBe('"document"."acl" && ARRAY[$1, $2, $3]::text[]') + expect(params).toEqual(['pub', 's:confluence:-:557058:abc', 'ws']) + for (const param of params) expect(Array.isArray(param)).toBe(false) + }) + + it('renders the workspace pair for actorless callers', () => { + const { sql, params } = render( + knowledgeAccessCondition({ kind: 'workspace', tokens: ['pub', 'ws'] }) + ) + expect(sql).toBe('"document"."acl" && ARRAY[$1, $2]::text[]') + expect(params).toEqual(['pub', 'ws']) + }) + + it('denies everything for an empty token set', () => { + expect(render(knowledgeAccessCondition({ kind: 'user', userId: 'u', tokens: [] })).sql).toBe( + 'false' + ) + }) + + it('exempts only the branded system scope', () => { + expect(render(knowledgeAccessCondition(SYSTEM_ACCESS_SCOPE)).sql).toBe('true') + }) +}) diff --git a/apps/sim/lib/knowledge/access/predicate.ts b/apps/sim/lib/knowledge/access/predicate.ts new file mode 100644 index 00000000000..2612e75938f --- /dev/null +++ b/apps/sim/lib/knowledge/access/predicate.ts @@ -0,0 +1,31 @@ +import { document } from '@sim/db/schema' +import { type SQL, sql } from 'drizzle-orm' +import type { KnowledgeAccessScope, SystemAccessScope } from '@/lib/knowledge/access/types' + +/** + * The single read-side access predicate: the document's ACL overlaps the + * caller's token set. Tokens are bound as scalars and assembled with + * `ARRAY[...]` because the shared pool runs with `fetch_types: false`, under + * which a JS array bound as one parameter fails at execution (see + * packages/db/db.ts). A literal array also keeps the planner's statistics on + * `acl` usable, which is what lets it choose the GIN index for a selective set. + * + * The system scope is the only exemption and renders as `true`. + */ +export function knowledgeAccessCondition(scope: KnowledgeAccessScope | SystemAccessScope): SQL { + if (scope.kind === 'system') return sql`true` + if (scope.tokens.length === 0) return sql`false` + return sql`${document.acl} && ${textArrayLiteral(scope.tokens)}` +} + +/** + * A `text[]` literal assembled from scalar binds, for comparing against an + * ACL column. Every place that compares ACLs builds its array this way, for + * the `fetch_types: false` reason above. + */ +export function textArrayLiteral(values: readonly string[]): SQL { + return sql`ARRAY[${sql.join( + values.map((value) => sql`${value}`), + sql`, ` + )}]::text[]` +} diff --git a/apps/sim/lib/knowledge/access/scope.test.ts b/apps/sim/lib/knowledge/access/scope.test.ts new file mode 100644 index 00000000000..759817878f3 --- /dev/null +++ b/apps/sim/lib/knowledge/access/scope.test.ts @@ -0,0 +1,221 @@ +/** + * @vitest-environment node + */ +import type { Principal } from '@sim/auth/principal' +import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockMemberAccessAvailable, mockCheckWorkspaceAccess } = vi.hoisted(() => ({ + mockMemberAccessAvailable: vi.fn(async () => true), + mockCheckWorkspaceAccess: vi.fn(async () => ({ hasAccess: true })), +})) + +vi.mock('@/lib/knowledge/access/availability', () => ({ + isKnowledgeMemberAccessAvailable: mockMemberAccessAvailable, +})) +vi.mock('@/lib/workspaces/permissions/utils', () => ({ + checkWorkspaceAccess: mockCheckWorkspaceAccess, +})) + +import { + createKnowledgeAccessProvider, + resolveKnowledgeAccessScope, + WORKSPACE_ACCESS_SCOPE, +} from '@/lib/knowledge/access/scope' + +const SESSION: Principal = { kind: 'session', userId: 'user-1', sessionId: 'session-1' } +const WORKSPACE = { workspaceId: 'ws-1' } + +function queueSubjects(rows: Array>) { + queueTableRows(schemaMock.user, rows) +} + +describe('resolveKnowledgeAccessScope', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + it('gives a person the workspace pair plus one token per active managed credential', async () => { + queueSubjects([ + { providerId: 'confluence', providerTenantId: null, providerSubjectId: '557058:abc' }, + { providerId: 'google-drive', providerTenantId: 'acme.com', providerSubjectId: '42' }, + { providerId: 'confluence', providerTenantId: null, providerSubjectId: '557058:abc' }, + ]) + + const scope = await resolveKnowledgeAccessScope(SESSION, WORKSPACE) + + expect(scope).toEqual({ + kind: 'user', + userId: 'user-1', + tokens: ['pub', 's:confluence:-:557058:abc', 's:google-drive:acme.com:42', 'ws'], + }) + expect(dbChainMockFns.leftJoin).toHaveBeenCalledTimes(3) + }) + + it('grants no member token to someone who is no longer in the workspace', async () => { + mockCheckWorkspaceAccess.mockResolvedValueOnce({ hasAccess: false }) + await expect(resolveKnowledgeAccessScope(SESSION, WORKSPACE)).resolves.toEqual({ + kind: 'user', + userId: 'user-1', + tokens: ['pub', 'ws'], + }) + expect(dbChainMockFns.select).not.toHaveBeenCalled() + }) + + it('grants no member token where per-member access is off, whatever the person holds', async () => { + mockMemberAccessAvailable.mockResolvedValueOnce(false) + queueSubjects([ + { providerId: 'google-drive', providerTenantId: 'acme.com', providerSubjectId: '42' }, + ]) + await expect(resolveKnowledgeAccessScope(SESSION, WORKSPACE)).resolves.toEqual({ + kind: 'user', + userId: 'user-1', + tokens: ['pub', 'ws'], + }) + }) + + it('falls back to the workspace pair for a person with no credential, and for one who is unverified or unknown', async () => { + queueSubjects([{ providerId: null, providerTenantId: null, providerSubjectId: null }]) + await expect(resolveKnowledgeAccessScope(SESSION, WORKSPACE)).resolves.toEqual({ + kind: 'user', + userId: 'user-1', + tokens: ['pub', 'ws'], + }) + + resetDbChainMock() + queueSubjects([]) + await expect(resolveKnowledgeAccessScope(SESSION, WORKSPACE)).resolves.toEqual({ + kind: 'user', + userId: 'user-1', + tokens: ['pub', 'ws'], + }) + }) + + it('skips a malformed credential row instead of failing the read', async () => { + queueSubjects([ + { providerId: 'a:b', providerTenantId: null, providerSubjectId: 'x' }, + { providerId: 'slack', providerTenantId: 'T1', providerSubjectId: 'U1' }, + ]) + await expect(resolveKnowledgeAccessScope(SESSION, WORKSPACE)).resolves.toEqual({ + kind: 'user', + userId: 'user-1', + tokens: ['pub', 's:slack:T1:U1', 'ws'], + }) + }) + + it('does not query for a legacy personal knowledge base', async () => { + await expect(resolveKnowledgeAccessScope(SESSION, {})).resolves.toEqual({ + kind: 'user', + userId: 'user-1', + tokens: ['pub', 'ws'], + }) + expect(dbChainMockFns.select).not.toHaveBeenCalled() + }) + + it.each<[string, Principal]>([ + ['a workspace API key', { kind: 'workspace_api_key', workspaceId: 'ws-1', keyId: 'key-1' }], + [ + 'a scheduled run', + { kind: 'system', serviceId: 'schedule', workspaceId: 'ws-1', workflowId: 'wf-1' }, + ], + [ + 'a webhook run with an external subject', + { + kind: 'system', + serviceId: 'webhook', + workspaceId: 'ws-1', + workflowId: 'wf-1', + webhookId: 'wh-1', + provider: 'slack', + subject: { kind: 'external_user', provider: 'slack', tenantId: 'T1', subjectId: 'U1' }, + }, + ], + [ + 'an executor run whose trigger was a workspace key', + { + kind: 'delegated', + serviceId: 'executor', + workspaceId: 'ws-1', + delegationId: 'd-1', + audience: 'sim:knowledge', + issuedAt: 0, + expiresAt: 1, + delegationContext: { + principal: { kind: 'workspace_api_key', workspaceId: 'ws-1', keyId: 'key-1' }, + compatibilityActor: { userId: 'deployer' }, + currentWorkflow: { workflowId: 'wf-1', mode: 'deployment' }, + }, + } as unknown as Principal, + ], + ])('resolves %s to the workspace scope without a lookup', async (_label, principal) => { + await expect(resolveKnowledgeAccessScope(principal, WORKSPACE)).resolves.toBe( + WORKSPACE_ACCESS_SCOPE + ) + expect(dbChainMockFns.select).not.toHaveBeenCalled() + }) + + it('follows an executor delegation back to the person who triggered it', async () => { + queueSubjects([]) + const executor = { + kind: 'delegated', + serviceId: 'executor', + workspaceId: 'ws-1', + delegationId: 'd-1', + audience: 'sim:knowledge', + issuedAt: 0, + expiresAt: 1, + delegationContext: { + principal: SESSION, + currentWorkflow: { workflowId: 'wf-1', mode: 'draft' }, + }, + } as unknown as Principal + + await expect(resolveKnowledgeAccessScope(executor, WORKSPACE)).resolves.toMatchObject({ + kind: 'user', + userId: 'user-1', + }) + }) + + it('refuses a Credential Group enrollment principal', async () => { + await expect( + resolveKnowledgeAccessScope( + { + kind: 'credential_group_enrollment', + workspaceId: 'ws-1', + credentialGroupId: 'g', + enrollmentId: 'e', + email: 'a@b.c', + invitationTokenHash: 'h', + } as Principal, + WORKSPACE + ) + ).rejects.toThrow('cannot read knowledge documents') + }) +}) + +describe('createKnowledgeAccessProvider', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + it('resolves once per operation and shares the result', async () => { + queueSubjects([{ providerId: 'slack', providerTenantId: 'T1', providerSubjectId: 'U1' }]) + const provider = createKnowledgeAccessProvider(SESSION, WORKSPACE) + + const [first, second] = await Promise.all([provider.get(), provider.get()]) + + expect(first).toBe(second) + expect(dbChainMockFns.select).toHaveBeenCalledTimes(1) + }) + + it('retries after a failed lookup rather than caching the failure', async () => { + dbChainMockFns.where.mockRejectedValueOnce(new Error('connection reset')) + const provider = createKnowledgeAccessProvider(SESSION, WORKSPACE) + + await expect(provider.get()).rejects.toThrow('connection reset') + queueSubjects([]) + await expect(provider.get()).resolves.toMatchObject({ kind: 'user', tokens: ['pub', 'ws'] }) + }) +}) diff --git a/apps/sim/lib/knowledge/access/scope.ts b/apps/sim/lib/knowledge/access/scope.ts new file mode 100644 index 00000000000..3cae9340d40 --- /dev/null +++ b/apps/sim/lib/knowledge/access/scope.ts @@ -0,0 +1,177 @@ +import { type Principal, resolvePrincipalSubject } from '@sim/auth/principal' +import { db } from '@sim/db' +import { credential, credentialGroup, credentialGroupEnrollment, user } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { and, eq, inArray, sql } from 'drizzle-orm' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { LIVE_ENROLLMENT_STATUSES } from '@/lib/credential-groups/credentials' +import { isKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' +import { sortAccessTokens, subjectToken } from '@/lib/knowledge/access/tokens' +import { + type KnowledgeAccessProvider, + type KnowledgeAccessScope, + WORKSPACE_ACCESS_TOKENS, + type WorkspaceAccessScope, +} from '@/lib/knowledge/access/types' +import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils' + +const logger = createLogger('KnowledgeAccessScope') + +export const WORKSPACE_ACCESS_SCOPE: WorkspaceAccessScope = Object.freeze({ + kind: 'workspace', + tokens: WORKSPACE_ACCESS_TOKENS, +}) + +/** Enrollment states under which a credential-group membership counts as live. */ + +export interface KnowledgeAccessScopeContext { + /** Undefined only for a legacy personal knowledge base, which cannot own connectors. */ + workspaceId?: string +} + +/** + * The tokens a person holds in a workspace: the workspace pair plus one `s:` + * token per active managed credential bound to them through a credential-group + * enrollment. The person must be email-verified — the enrollment binding is by + * email, and an unverified address must not inherit grants made to whoever + * really owns it. Nothing here is cached: revoking or suspending a credential + * is visible on the next read. + */ +async function loadUserAccessTokens( + userId: string, + workspaceId: string | undefined +): Promise { + if (!workspaceId) return [...WORKSPACE_ACCESS_TOKENS] + + /** + * Member tokens belong to current workspace members. Resolved before any + * document is looked up, so someone who left the workspace but still holds + * a managed credential cannot learn which documents their old tokens match. + */ + const workspaceAccess = await checkWorkspaceAccess(workspaceId, userId) + if (!workspaceAccess.hasAccess) return [...WORKSPACE_ACCESS_TOKENS] + /** + * A member token only counts where permission-aware knowledge is on, so + * turning the feature off hides every member-scoped document at once — on + * the next read, before any run has suspended anyone — rather than leaving + * enrolled members reading them until a run happens to land. Read first, so + * a workspace without the feature never pays for the enrollment join. + */ + if (!(await isKnowledgeMemberAccessAvailable({ workspaceId }))) { + return [...WORKSPACE_ACCESS_TOKENS] + } + + const rows = await db + .select({ + providerId: credential.providerId, + providerTenantId: credential.providerTenantId, + providerSubjectId: credential.providerSubjectId, + }) + .from(user) + .leftJoin( + credentialGroupEnrollment, + and( + eq( + credentialGroupEnrollment.email, + sql`COALESCE(${user.normalizedEmail}, lower(btrim(${user.email})))` + ), + inArray(credentialGroupEnrollment.status, [...LIVE_ENROLLMENT_STATUSES]) + ) + ) + .leftJoin( + credentialGroup, + and( + eq(credentialGroup.id, credentialGroupEnrollment.credentialGroupId), + eq(credentialGroup.status, 'active') + ) + ) + .leftJoin( + credential, + and( + eq(credential.credentialGroupEnrollmentId, credentialGroupEnrollment.id), + eq(credential.workspaceId, workspaceId), + eq(credential.type, 'managed_oauth'), + eq(credential.managedOauthStatus, 'active'), + /** The option must still be live, exactly as the member engine requires. */ + sql`EXISTS ( + SELECT 1 FROM jsonb_array_elements(${credentialGroup.options}) AS option + WHERE option->>'id' = ${credential.credentialGroupOptionId} + AND option->>'status' = 'active' + )` + ) + ) + .where(and(eq(user.id, userId), eq(user.emailVerified, true))) + + const subjectTokens = new Set() + for (const row of rows) { + if (!row.providerSubjectId) continue + try { + subjectTokens.add(subjectToken(row)) + } catch (error) { + logger.warn('Skipping malformed managed credential subject', { + userId, + workspaceId, + providerId: row.providerId, + error: getErrorMessage(error), + }) + } + } + return sortAccessTokens(new Set([...WORKSPACE_ACCESS_TOKENS, ...subjectTokens])) +} + +/** + * Resolves what a principal may read. A principal with a person behind it gets + * that person's tokens; everything actorless — workspace API keys, scheduled, + * webhook, chat, and MCP runs — gets the workspace pair, by policy. Never + * consults a compatibility actor: a scheduled run must not inherit its + * deployer's private documents. + */ +export async function resolveKnowledgeAccessScope( + principal: Principal, + context: KnowledgeAccessScopeContext +): Promise { + if (principal.kind === 'credential_group_enrollment') { + throw new OrchestrationError( + 'forbidden', + 'Credential Group enrollments cannot read knowledge documents' + ) + } + const subject = resolvePrincipalSubject(principal) + if (subject?.kind !== 'sim_user') return WORKSPACE_ACCESS_SCOPE + return { + kind: 'user', + userId: subject.userId, + tokens: await loadUserAccessTokens(subject.userId, context.workspaceId), + } +} + +/** + * The scope of a person identified only by user id — the shape session-backed + * routes outside the application layer have in hand. Never call this with a + * user id that stands in for an actorless run (a workflow owner, a billing + * owner); those callers use {@link WORKSPACE_ACCESS_SCOPE}. + */ +export async function resolveUserKnowledgeAccessScope( + userId: string, + workspaceId: string | undefined +): Promise { + return { kind: 'user', userId, tokens: await loadUserAccessTokens(userId, workspaceId) } +} + +/** Memoises {@link resolveKnowledgeAccessScope} for one operation; a failed lookup is retried on the next call. */ +export function createKnowledgeAccessProvider( + principal: Principal, + context: KnowledgeAccessScopeContext +): KnowledgeAccessProvider { + let pending: Promise | undefined + return { + get() { + pending ??= resolveKnowledgeAccessScope(principal, context).catch((error: unknown) => { + pending = undefined + throw error + }) + return pending + }, + } +} diff --git a/apps/sim/lib/knowledge/access/tokens.test.ts b/apps/sim/lib/knowledge/access/tokens.test.ts new file mode 100644 index 00000000000..de56395ad7d --- /dev/null +++ b/apps/sim/lib/knowledge/access/tokens.test.ts @@ -0,0 +1,95 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + ACCESS_TOKEN_PATTERN, + isAccessToken, + sortAccessTokens, + subjectToken, +} from '@/lib/knowledge/access/tokens' + +describe('access token shape', () => { + it.each([ + 'ws', + 'pub', + 'link', + 'u:alice@acme.com', + 's:confluence:-:557058:9f2b-uuid', + 's:google-drive:acme.com:1029384756', + 'g:sharepoint:tid-guid:sp:host,site,web:12', + ])('accepts %s', (token) => { + expect(isAccessToken(token)).toBe(true) + }) + + it.each([ + 'u:Alice@acme.com', + 's:confluence:557058', + 'x:foo', + '', + 'ws\npub', + 's::-:subject', + 'u:alice', + ])('rejects %j', (token) => { + expect(isAccessToken(token)).toBe(false) + }) + + it('mirrors the database check constraint per element', () => { + expect(ACCESS_TOKEN_PATTERN.source).toContain('[gs]:[^\\n:]+:[^\\n:]+:[^\\n]+') + }) +}) + +describe('subjectToken', () => { + it('derives the token from the credential row, substituting the no-tenant segment', () => { + expect( + subjectToken({ + providerId: 'confluence', + providerTenantId: null, + providerSubjectId: '557058:9f2b-uuid', + }) + ).toBe('s:confluence:-:557058:9f2b-uuid') + expect( + subjectToken({ + providerId: 'google-drive', + providerTenantId: 'acme.com', + providerSubjectId: '1029384756', + }) + ).toBe('s:google-drive:acme.com:1029384756') + }) + + it('treats an empty tenant like a missing one', () => { + expect( + subjectToken({ providerId: 'slack', providerTenantId: '', providerSubjectId: 'U1' }) + ).toBe('s:slack:-:U1') + }) + + it('fails loudly on a credential that cannot identify a person', () => { + expect(() => + subjectToken({ providerId: 'confluence', providerTenantId: null, providerSubjectId: null }) + ).toThrow('requires a provider id') + expect(() => + subjectToken({ providerId: null, providerTenantId: null, providerSubjectId: 'x' }) + ).toThrow('requires a provider id') + expect(() => + subjectToken({ providerId: 'a:b', providerTenantId: null, providerSubjectId: 'x' }) + ).toThrow('cannot contain ":"') + expect(() => + subjectToken({ providerId: 'slack', providerTenantId: 'T:1', providerSubjectId: 'x' }) + ).toThrow('cannot contain ":"') + }) +}) + +describe('sortAccessTokens', () => { + it('sorts by code unit and dedupes', () => { + expect(sortAccessTokens(['ws', 'pub', 's:b:-:1', 'pub', 's:B:-:1'])).toEqual([ + 'pub', + 's:B:-:1', + 's:b:-:1', + 'ws', + ]) + }) + + it('never uses locale ordering', () => { + expect(sortAccessTokens(['s:x:-:b', 's:x:-:B'])).toEqual(['s:x:-:B', 's:x:-:b']) + }) +}) diff --git a/apps/sim/lib/knowledge/access/tokens.ts b/apps/sim/lib/knowledge/access/tokens.ts new file mode 100644 index 00000000000..ff4cedfc63b --- /dev/null +++ b/apps/sim/lib/knowledge/access/tokens.ts @@ -0,0 +1,62 @@ +import { WORKSPACE_ACCESS_TOKEN } from '@/lib/knowledge/access/types' + +/** + * Shape of one access token, mirroring `doc_acl_token_shape_check` in the + * database. `u:` carries a lowercase email; `s:` and `g:` carry three + * colon-separated segments — provider, tenant, subject or group id — where only + * the last may itself contain colons (Atlassian account ids do). + */ +export const ACCESS_TOKEN_PATTERN = + /^(ws|pub|link|u:[^\nA-Z]+@[^\nA-Z]+|[gs]:[^\n:]+:[^\n:]+:[^\n]+)$/ + +/** Stands in for a provider that reports no tenant, so the token keeps four segments. */ +export const NO_TENANT_SEGMENT = '-' + +/** The ACL of a document only the workspace's uploads path or a workspace-mode connector wrote. */ +export const WORKSPACE_ACL: readonly string[] = Object.freeze([WORKSPACE_ACCESS_TOKEN]) + +/** The ACL of a document nobody may read. */ +export const EMPTY_ACL: readonly string[] = Object.freeze([]) + +export function isAccessToken(value: string): boolean { + return ACCESS_TOKEN_PATTERN.test(value) +} + +export interface SubjectCredential { + providerId: string | null + providerTenantId: string | null + providerSubjectId: string | null +} + +/** + * The identity token of a person by the provider-attested subject on their + * managed credential. Both the writer (a members-mode crawl) and the reader + * (scope resolution) derive it from the same `credential` row, so no + * source-side id format is ever compared to another. + */ +export function subjectToken(credential: SubjectCredential): string { + const { providerId, providerSubjectId } = credential + if (!providerId || !providerSubjectId) { + throw new Error('A subject token requires a provider id and a provider subject id') + } + const tenant = credential.providerTenantId || NO_TENANT_SEGMENT + if (providerId.includes(':') || tenant.includes(':')) { + throw new Error('Provider and tenant segments of a subject token cannot contain ":"') + } + const token = `s:${providerId}:${tenant}:${providerSubjectId}` + if (!isAccessToken(token)) { + throw new Error(`Subject token is malformed: ${token}`) + } + return token +} + +/** + * Canonical ordering for every ACL and token set: code-unit order, never + * locale-aware, so two writers produce byte-identical arrays and Postgres array + * comparison stays meaningful. + */ +export function sortAccessTokens(tokens: Iterable): string[] { + const unique = [...new Set(tokens)] + unique.sort((left, right) => (left < right ? -1 : left > right ? 1 : 0)) + return unique +} diff --git a/apps/sim/lib/knowledge/access/types.ts b/apps/sim/lib/knowledge/access/types.ts new file mode 100644 index 00000000000..2d3346d6b91 --- /dev/null +++ b/apps/sim/lib/knowledge/access/types.ts @@ -0,0 +1,57 @@ +/** Held by every principal in the workspace; the default ACL of an upload. */ +export const WORKSPACE_ACCESS_TOKEN = 'ws' as const + +/** Held by every principal; a document the source itself makes public. */ +export const PUBLIC_ACCESS_TOKEN = 'pub' as const + +/** + * The token set of a caller with no person behind it — a workspace API key, a + * scheduled or webhook run, chat, MCP — and the base every person's set + * extends. Sorted, like every ACL, so array comparisons are meaningful. + */ +export const WORKSPACE_ACCESS_TOKENS = [PUBLIC_ACCESS_TOKEN, WORKSPACE_ACCESS_TOKEN] as const + +export interface WorkspaceAccessScope { + kind: 'workspace' + tokens: typeof WORKSPACE_ACCESS_TOKENS +} + +export interface UserAccessScope { + kind: 'user' + userId: string + /** `pub`, `ws`, and one `s:` token per active managed credential the person holds here. */ + tokens: readonly string[] +} + +/** + * What the calling principal may read, expressed as the tokens it holds. Every + * document loader takes one; there is no way to read a document without it. + */ +export type KnowledgeAccessScope = WorkspaceAccessScope | UserAccessScope + +/** + * Lazily resolves the scope for one authorized operation. Created by the + * knowledge context resolvers and attached to the use-case context, so a + * write-only operation never pays for the membership lookup and a read + * resolves it exactly once. + */ +export interface KnowledgeAccessProvider { + get(): Promise +} + +declare const systemAccessScopeBrand: unique symbol + +/** + * The one exemption from access filtering: a background job acting on rows it + * owns (document processing, connector sync). It is a branded type so it cannot + * be assembled from a literal, and this module is its only source, so every + * caller is one grep away. Never construct it on a request path. + */ +export interface SystemAccessScope { + readonly kind: 'system' + readonly [systemAccessScopeBrand]: true +} + +export const SYSTEM_ACCESS_SCOPE: SystemAccessScope = Object.freeze({ + kind: 'system', +}) as SystemAccessScope diff --git a/apps/sim/lib/knowledge/api/internal-route.test.ts b/apps/sim/lib/knowledge/api/internal-route.test.ts index 4d69fea1e32..de40cb0269e 100644 --- a/apps/sim/lib/knowledge/api/internal-route.test.ts +++ b/apps/sim/lib/knowledge/api/internal-route.test.ts @@ -11,6 +11,7 @@ import { import { internalKnowledgeProvenanceUserId, resolveInternalKnowledgeBillingAttribution, + toInternalKnowledgeConnector, } from '@/lib/knowledge/api/internal-route' import { resolveKnowledgeAttributedUserId } from '@/lib/knowledge/application/billing' @@ -127,3 +128,42 @@ describe('internal Knowledge execution attribution', () => { ).rejects.toThrow('does not match the authenticated request scope') }) }) + +describe('toInternalKnowledgeConnector', () => { + const row = { + id: 'connector-1', + knowledgeBaseId: 'kb-1', + connectorType: 'google_drive', + credentialId: null, + sourceConfig: {}, + syncMode: null, + syncIntervalMinutes: 60, + status: 'active' as const, + lastSyncAt: null, + lastSyncError: null, + lastSyncDocCount: null, + nextSyncAt: null, + consecutiveFailures: 0, + accessMode: 'members' as const, + credentialGroupId: 'group-1', + credentialGroupOptionId: 'option-1', + memberSyncStatus: 'idle' as const, + lastMemberSyncAt: null, + nextMemberSyncAt: null, + lastMemberSyncError: null, + memberSyncConsecutiveFailures: 0, + accessRewritePending: false, + createdAt: new Date('2026-09-01T00:00:00Z'), + updatedAt: new Date('2026-09-01T00:00:00Z'), + } + + it('presents a mutation result, which carries no viewer membership, as null', () => { + expect(toInternalKnowledgeConnector(row).viewerMembership).toBeNull() + }) + + it('keeps the membership a read resolved for the viewer', () => { + expect( + toInternalKnowledgeConnector({ ...row, viewerMembership: 'invited' }).viewerMembership + ).toBe('invited') + }) +}) diff --git a/apps/sim/lib/knowledge/api/internal-route.ts b/apps/sim/lib/knowledge/api/internal-route.ts index 21fe9b0510f..43d91dafd40 100644 --- a/apps/sim/lib/knowledge/api/internal-route.ts +++ b/apps/sim/lib/knowledge/api/internal-route.ts @@ -10,6 +10,7 @@ import { type ChunkData, chunkDataSchema } from '@/lib/api/contracts/knowledge/c import { type ConnectorData, type ConnectorDetailData, + type ConnectorMemberSummary, connectorDataSchema, connectorDetailDataSchema, } from '@/lib/api/contracts/knowledge/connectors' @@ -132,14 +133,21 @@ export function toInternalKnowledgeConnector< updatedAt: Date | string lastSyncAt: Date | string | null nextSyncAt: Date | string | null + lastMemberSyncAt: Date | string | null + nextMemberSyncAt: Date | string | null + viewerMembership?: ConnectorData['viewerMembership'] }, >(connector: T): ConnectorData { return connectorDataSchema.parse({ + /** A mutation answers with the row alone; only a viewer's read carries their membership. */ + viewerMembership: null, ...connector, createdAt: serializeDate(connector.createdAt), updatedAt: serializeDate(connector.updatedAt), lastSyncAt: serializeNullableDate(connector.lastSyncAt), nextSyncAt: serializeNullableDate(connector.nextSyncAt), + lastMemberSyncAt: serializeNullableDate(connector.lastMemberSyncAt), + nextMemberSyncAt: serializeNullableDate(connector.nextMemberSyncAt), }) } @@ -150,6 +158,12 @@ export function toInternalKnowledgeConnectorDetail< completedAt: Date | string | null [key: string]: unknown }> + memberSyncLogs: Array<{ + startedAt: Date | string + completedAt: Date | string | null + [key: string]: unknown + }> + members: ConnectorMemberSummary }, >(connector: T): ConnectorDetailData { return connectorDetailDataSchema.parse({ @@ -159,6 +173,12 @@ export function toInternalKnowledgeConnectorDetail< startedAt: serializeDate(log.startedAt), completedAt: serializeNullableDate(log.completedAt), })), + memberSyncLogs: connector.memberSyncLogs.map((log) => ({ + ...log, + startedAt: serializeDate(log.startedAt), + completedAt: serializeNullableDate(log.completedAt), + })), + members: connector.members, }) } diff --git a/apps/sim/lib/knowledge/application/add-workspace-files.ts b/apps/sim/lib/knowledge/application/add-workspace-files.ts index f7deb79d44d..584f9f52834 100644 --- a/apps/sim/lib/knowledge/application/add-workspace-files.ts +++ b/apps/sim/lib/knowledge/application/add-workspace-files.ts @@ -122,8 +122,10 @@ async function prepareWorkspaceFile( export const addWorkspaceFilesToKnowledgeBase = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.addWorkspaceFiles, async resolveContext({ + principal, input, }: { + principal: Principal input: AddWorkspaceFilesToKnowledgeBaseInput }): Promise { const fileReferences = requireBoundedKnowledgeBatch( @@ -132,7 +134,7 @@ export const addWorkspaceFilesToKnowledgeBase = defineAuthorizedKnowledgeUseCase ADD_WORKSPACE_FILES_COST_POLICY.maxItems ) return { - ...(await resolveActiveKnowledgeBaseContext(input)), + ...(await resolveActiveKnowledgeBaseContext(input, principal)), fileReferences, } }, diff --git a/apps/sim/lib/knowledge/application/bulk.ts b/apps/sim/lib/knowledge/application/bulk.ts index f838260ad78..6ae49caaa83 100644 --- a/apps/sim/lib/knowledge/application/bulk.ts +++ b/apps/sim/lib/knowledge/application/bulk.ts @@ -1,4 +1,5 @@ import { AuditAction, AuditResourceType } from '@sim/audit' +import type { Principal } from '@sim/auth/principal' import { createLogger } from '@sim/logger' import { authorizeWorkspaceOperation } from '@/lib/core/application' import { classifyBulkItemError } from '@/lib/core/application/bulk-items' @@ -120,6 +121,7 @@ async function resolveBulkKnowledgeContext( async function runKnowledgeItems( knowledgeBaseIds: readonly string[], workspace: KnowledgeWorkspaceContext, + principal: Principal, covered: ReadonlySet, authorize: (canonical: ActiveKnowledgeBaseContext) => Promise, apply: (canonical: ActiveKnowledgeBaseContext) => Promise, @@ -129,7 +131,11 @@ async function runKnowledgeItems( for (const knowledgeBaseId of knowledgeBaseIds) { let knowledgeBaseName = knowledgeBaseId try { - const canonical = await resolveActiveKnowledgeBaseInWorkspace(knowledgeBaseId, workspace) + const canonical = await resolveActiveKnowledgeBaseInWorkspace( + knowledgeBaseId, + workspace, + principal + ) knowledgeBaseName = canonical.knowledgeBase.name const folderId = canonical.knowledgeBase.folderId if (folderId && covered.has(folderId)) { @@ -169,8 +175,13 @@ async function runKnowledgeItems( export const bulkMoveKnowledgeItems = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.bulkMoveItems, - resolveContext: ({ input }: { input: BulkMoveKnowledgeItemsInput }) => - resolveBulkKnowledgeContext(input, BULK_MOVE_KNOWLEDGE_ITEMS_COST_POLICY.maxItems), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: BulkMoveKnowledgeItemsInput + }) => resolveBulkKnowledgeContext(input, BULK_MOVE_KNOWLEDGE_ITEMS_COST_POLICY.maxItems), async execute({ principal, input, context }): Promise { /** * The destination check and the folder plan read different rows and share @@ -219,6 +230,7 @@ export const bulkMoveKnowledgeItems = defineAuthorizedKnowledgeUseCase({ const terminalError = await runKnowledgeItems( context.knowledgeBaseIds, context, + principal, plan.covered, (canonical) => authorizeWorkspaceOperation(principal, knowledgeOperations.bulkMoveItems, canonical, { @@ -305,8 +317,13 @@ export const bulkMoveKnowledgeItems = defineAuthorizedKnowledgeUseCase({ export const bulkDeleteKnowledgeItems = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.bulkDeleteItems, - resolveContext: ({ input }: { input: BulkDeleteKnowledgeItemsInput }) => - resolveBulkKnowledgeContext(input, BULK_DELETE_KNOWLEDGE_ITEMS_COST_POLICY.maxItems), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: BulkDeleteKnowledgeItemsInput + }) => resolveBulkKnowledgeContext(input, BULK_DELETE_KNOWLEDGE_ITEMS_COST_POLICY.maxItems), async execute({ principal, context }): Promise { const plan = await planFolderSelection( context.workspaceId, @@ -321,6 +338,7 @@ export const bulkDeleteKnowledgeItems = defineAuthorizedKnowledgeUseCase({ const terminalError = await runKnowledgeItems( context.knowledgeBaseIds, context, + principal, plan.covered, (canonical) => authorizeWorkspaceOperation(principal, knowledgeOperations.bulkDeleteItems, canonical, { diff --git a/apps/sim/lib/knowledge/application/chunks.test.ts b/apps/sim/lib/knowledge/application/chunks.test.ts index fffc0cf7d06..6e1a0177b75 100644 --- a/apps/sim/lib/knowledge/application/chunks.test.ts +++ b/apps/sim/lib/knowledge/application/chunks.test.ts @@ -45,6 +45,7 @@ vi.mock('@/lib/knowledge/model-input-provenance', () => ({ vi.mock('@/providers/utils', () => ({ calculateCost: vi.fn() })) import { ForbiddenOperationError } from '@/lib/core/application/forbidden' +import { WORKSPACE_ACCESS_SCOPE } from '@/lib/knowledge/access/scope' import { KnowledgeDocumentNotReadyError } from '@/lib/knowledge/application/chunk-errors' import { bulkUpdateKnowledgeChunks, listKnowledgeChunks } from '@/lib/knowledge/application/chunks' @@ -57,6 +58,7 @@ describe('knowledge chunk application use cases', () => { workspaceOrganizationId: null, allowPersonalApiKeys: true, billedAccountUserId: 'billing-owner-1', + access: { get: async () => WORKSPACE_ACCESS_SCOPE }, knowledgeBaseId: 'knowledge-1', knowledgeBase: { id: 'knowledge-1' }, documentId: 'document-1', @@ -85,6 +87,7 @@ describe('knowledge chunk application use cases', () => { workspaceOrganizationId: null, allowPersonalApiKeys: true, billedAccountUserId: 'billing-owner-1', + access: { get: async () => WORKSPACE_ACCESS_SCOPE }, knowledgeBaseId: 'knowledge-1', knowledgeBase: { id: 'knowledge-1' }, documentId: 'document-1', @@ -108,7 +111,8 @@ describe('knowledge chunk application use cases', () => { expect(mocks.queryChunks).toHaveBeenCalledWith( 'document-1', expect.objectContaining({ cursorKeys: [3, 'chunk-3'] }), - expect.any(String) + expect.any(String), + WORKSPACE_ACCESS_SCOPE ) expect(result.nextCursorKeys).toBeNull() }) @@ -125,6 +129,7 @@ describe('knowledge chunk application use cases', () => { workspaceOrganizationId: null, allowPersonalApiKeys: true, billedAccountUserId: 'billing-owner-1', + access: { get: async () => WORKSPACE_ACCESS_SCOPE }, knowledgeBaseId: 'knowledge-1', knowledgeBase: { id: 'knowledge-1' }, documentId: 'document-1', diff --git a/apps/sim/lib/knowledge/application/chunks.ts b/apps/sim/lib/knowledge/application/chunks.ts index 9e22dbddf0f..9da76b789f8 100644 --- a/apps/sim/lib/knowledge/application/chunks.ts +++ b/apps/sim/lib/knowledge/application/chunks.ts @@ -1,3 +1,4 @@ +import type { Principal } from '@sim/auth/principal' import { createLogger } from '@sim/logger' import { ForbiddenOperationError } from '@/lib/core/application/forbidden' import { OrchestrationError } from '@/lib/core/orchestration/types' @@ -102,8 +103,13 @@ function documentTags(context: ActiveKnowledgeDocumentContext) { export const listKnowledgeChunks = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.listChunks, - resolveContext: ({ input }: { input: ListKnowledgeChunksInput }) => - resolveCanonicalActiveKnowledgeDocumentContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: ListKnowledgeChunksInput + }) => resolveCanonicalActiveKnowledgeDocumentContext(input, principal), async execute({ input, context }) { requireChunkReadable(context) const { @@ -112,15 +118,20 @@ export const listKnowledgeChunks = defineAuthorizedKnowledgeUseCase({ assertedWorkspaceId: _scope, ...filters } = input - const result = await queryChunks(documentId, filters, generateRequestId()) + const result = await queryChunks( + documentId, + filters, + generateRequestId(), + await context.access.get() + ) return { ...result, workspaceId: context.workspaceId, documentId } }, }) export const readKnowledgeChunk = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.readChunk, - resolveContext: ({ input }: { input: KnowledgeChunkInput }) => - resolveActiveKnowledgeChunkContext(input), + resolveContext: ({ principal, input }: { principal: Principal; input: KnowledgeChunkInput }) => + resolveActiveKnowledgeChunkContext(input, principal), async execute({ context }) { requireChunkReadable(context) return { @@ -133,8 +144,13 @@ export const readKnowledgeChunk = defineAuthorizedKnowledgeUseCase({ export const createKnowledgeChunk = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.createChunk, - resolveContext: ({ input }: { input: CreateKnowledgeChunkInput }) => - resolveCanonicalActiveKnowledgeDocumentContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: CreateKnowledgeChunkInput + }) => resolveCanonicalActiveKnowledgeDocumentContext(input, principal), async execute({ principal, input, context }) { requireChunkWritable(context) if (context.document.processingStatus === 'failed') { @@ -195,8 +211,13 @@ export const createKnowledgeChunk = defineAuthorizedKnowledgeUseCase({ export const updateKnowledgeChunk = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.updateChunk, - resolveContext: ({ input }: { input: UpdateKnowledgeChunkInput }) => - resolveActiveKnowledgeChunkContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: UpdateKnowledgeChunkInput + }) => resolveActiveKnowledgeChunkContext(input, principal), async execute({ principal, input, context }) { requireChunkReadable(context) requireChunkWritable(context) @@ -226,8 +247,8 @@ export const updateKnowledgeChunk = defineAuthorizedKnowledgeUseCase({ export const deleteKnowledgeChunk = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.deleteChunk, - resolveContext: ({ input }: { input: KnowledgeChunkInput }) => - resolveActiveKnowledgeChunkContext(input), + resolveContext: ({ principal, input }: { principal: Principal; input: KnowledgeChunkInput }) => + resolveActiveKnowledgeChunkContext(input, principal), async execute({ context }) { requireChunkReadable(context) requireChunkWritable(context) @@ -238,8 +259,13 @@ export const deleteKnowledgeChunk = defineAuthorizedKnowledgeUseCase({ export const bulkUpdateKnowledgeChunks = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.bulkChunks, - resolveContext: ({ input }: { input: BulkKnowledgeChunksInput }) => - resolveCanonicalActiveKnowledgeDocumentContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: BulkKnowledgeChunksInput + }) => resolveCanonicalActiveKnowledgeDocumentContext(input, principal), async execute({ input, context }) { requireChunkWritable(context) const result = await batchChunkOperation( diff --git a/apps/sim/lib/knowledge/application/connector-access.ts b/apps/sim/lib/knowledge/application/connector-access.ts new file mode 100644 index 00000000000..3451da3499c --- /dev/null +++ b/apps/sim/lib/knowledge/application/connector-access.ts @@ -0,0 +1,220 @@ +import { AuditAction, AuditResourceType } from '@sim/audit' +import { type Principal, resolvePrincipalSubjectUserId } from '@sim/auth/principal' +import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { generateRequestId } from '@/lib/core/utils/request' +import { requireKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' +import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' +import { + resolveKnowledgeAttributedUserId, + resolveKnowledgeBillingAttribution, +} from '@/lib/knowledge/application/billing' +import { + requireConnectorWorkspaceId, + requireSuccessfulOutcome, + resolveConnectorCredentialAccessToken, +} from '@/lib/knowledge/application/connectors' +import { resolveActiveKnowledgeConnectorContext } from '@/lib/knowledge/application/contexts' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { createViewerConnectorEnrollmentLink } from '@/lib/knowledge/connectors/member-provisioning' +import { + performUpdateKnowledgeConnectorAccess, + resolveKnowledgeConnectorMembersBinding, +} from '@/lib/knowledge/orchestration/connector-access' +import { getKnowledgeConnector } from '@/lib/knowledge/orchestration/connectors' +import type { KnowledgeOperationSource } from '@/lib/knowledge/orchestration/shared' +import { getServiceConfigByProviderId, getServiceConfigByServiceId } from '@/lib/oauth' +import { getConnectorMeta } from '@/connectors/registry' +import type { ConnectorMeta } from '@/connectors/types' + +export interface StartKnowledgeConnectorMemberEnrollmentInput { + knowledgeBaseId: string + connectorId: string + assertedWorkspaceId?: string +} + +/** + * Hands a workspace member the link that connects their own account to a + * per-member connector, minted on demand so they never need the invitation + * email. Only widens what the member themselves can see. + */ +export const startKnowledgeConnectorMemberEnrollment = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.enrollConnectorMember, + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: StartKnowledgeConnectorMemberEnrollmentInput + }) => resolveActiveKnowledgeConnectorContext(input, principal), + async execute({ principal, context }) { + const workspaceId = requireConnectorWorkspaceId(context) + const userId = resolvePrincipalSubjectUserId(principal) + if (!userId) throw new OrchestrationError('forbidden', 'Sign in to connect your account') + const connector = await getKnowledgeConnector(context.knowledgeBaseId, context.connectorId) + if (!connector) throw new OrchestrationError('not_found', 'Connector not found') + if (connector.accessMode !== 'members' || !connector.credentialGroupId) { + throw new OrchestrationError('validation', 'This connector does not sync per member') + } + await requireKnowledgeMemberAccessAvailable({ workspaceId }) + const url = await createViewerConnectorEnrollmentLink({ + userId, + workspaceId, + credentialGroupId: connector.credentialGroupId, + }) + return { url } + }, +}) + +export interface UpdateKnowledgeConnectorAccessInput { + knowledgeBaseId: string + connectorId: string + assertedWorkspaceId?: string + accessMode: 'workspace' | 'members' + credentialGroupId?: string + credentialGroupOptionId?: string + /** Workspace mode: the credential the connector syncs as from now on. */ + credentialId?: string + source?: KnowledgeOperationSource + resolveBillingAttribution?(workspaceId: string): Promise +} + +/** + * Moves a connector between workspace and members mode. Admin only: members + * mode lets the connector crawl as every person enrolled in the option. + */ +export const updateKnowledgeConnectorAccess = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.updateConnectorAccess, + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: UpdateKnowledgeConnectorAccessInput + }) => resolveActiveKnowledgeConnectorContext(input, principal), + async execute({ principal, input, context, request }) { + const requestId = generateRequestId() + const workspaceId = requireConnectorWorkspaceId(context) + const actingUserId = resolveKnowledgeAttributedUserId(principal, context) + const connector = await getKnowledgeConnector(context.knowledgeBaseId, context.connectorId) + if (!connector) throw new OrchestrationError('not_found', 'Connector not found') + const connectorMeta = getConnectorMeta(connector.connectorType) + if (!connectorMeta) { + throw new OrchestrationError( + 'validation', + `Unknown connector type: ${connector.connectorType}` + ) + } + + const target = + input.accessMode === 'members' + ? { + accessMode: 'members' as const, + binding: await resolveKnowledgeConnectorMembersBinding({ + workspaceId, + connectorMeta, + binding: + input.credentialGroupId && input.credentialGroupOptionId + ? { + credentialGroupId: input.credentialGroupId, + credentialGroupOptionId: input.credentialGroupOptionId, + } + : null, + actingUserId, + sourceConfig: connector.sourceConfig as Record, + }), + } + : { + accessMode: 'workspace' as const, + credentialId: await requireUsableCredential({ + credentialId: input.credentialId, + connectorMeta, + workspaceId, + actingUserId, + requestId, + }), + } + + const outcome = await performUpdateKnowledgeConnectorAccess({ + knowledgeBase: { id: context.knowledgeBaseId, name: context.knowledgeBase.name, workspaceId }, + connectorId: context.connectorId, + target, + resolveBillingAttribution: () => + input.resolveBillingAttribution?.(workspaceId) ?? + resolveKnowledgeBillingAttribution(principal, context), + userId: actingUserId, + source: input.source ?? 'ui', + requestId, + request, + }) + requireSuccessfulOutcome(outcome, 'Knowledge connector access update failed') + return { connector: outcome.connector, changed: outcome.changed, workspaceId } + }, + projectAudit: ({ input, context, result }) => + result.changed + ? { + action: AuditAction.CONNECTOR_UPDATED, + resourceType: AuditResourceType.CONNECTOR, + resourceId: result.connector.id, + resourceName: result.connector.connectorType, + description: `Switched connector access to ${input.accessMode} mode for knowledge base "${context.knowledgeBase.name}"`, + metadata: { + source: input.source, + knowledgeBaseId: context.knowledgeBaseId, + knowledgeBaseName: context.knowledgeBase.name, + connectorType: result.connector.connectorType, + updatedFields: ['accessMode'], + accessMode: input.accessMode, + ...(input.credentialGroupId ? { credentialGroupId: input.credentialGroupId } : {}), + ...(input.credentialGroupOptionId + ? { credentialGroupOptionId: input.credentialGroupOptionId } + : {}), + }, + } + : [], +}) + +/** + * Workspace mode needs a credential the caller may use, of the connector's own + * provider, and one that yields a token, since the connector syncs as it from + * then on with no call to the source to catch a mismatch. Only an OAuth + * connector can change modes: an API-key connector has no account to sync per + * member. + */ +async function requireUsableCredential(input: { + credentialId: string | undefined + connectorMeta: Pick + workspaceId: string + actingUserId: string + requestId: string +}): Promise { + const { auth } = input.connectorMeta + if (auth.mode !== 'oauth') { + throw new OrchestrationError('validation', 'Only OAuth connectors can change access mode') + } + if (!input.credentialId) { + throw new OrchestrationError('validation', 'credentialId is required for workspace mode') + } + const service = + getServiceConfigByServiceId(auth.provider) ?? getServiceConfigByProviderId(auth.provider) + if (!service) { + throw new OrchestrationError( + 'validation', + `${input.connectorMeta.name} has no OAuth service to validate the credential against` + ) + } + const token = await resolveConnectorCredentialAccessToken({ + credentialId: input.credentialId, + workspaceId: input.workspaceId, + actingUserId: input.actingUserId, + requestId: input.requestId, + service, + }) + if (!token) { + throw new OrchestrationError( + 'validation', + 'Credential has no access token. Please reconnect your account.' + ) + } + return input.credentialId +} diff --git a/apps/sim/lib/knowledge/application/connectors.test.ts b/apps/sim/lib/knowledge/application/connectors.test.ts index 599f7f42ba3..6425cb7bf07 100644 --- a/apps/sim/lib/knowledge/application/connectors.test.ts +++ b/apps/sim/lib/knowledge/application/connectors.test.ts @@ -22,6 +22,7 @@ const mocks = vi.hoisted(() => ({ validateConnectorConfig: vi.fn(), recordAudit: vi.fn(), getUserPermissionConfig: vi.fn(), + resolveMembersBinding: vi.fn(), })) vi.mock('@sim/audit', () => ({ @@ -51,6 +52,10 @@ vi.mock('@/lib/knowledge/application/contexts', () => ({ resolveActiveKnowledgeConnectorContext: mocks.resolveConnector, })) +vi.mock('@/lib/knowledge/orchestration/connector-access', () => ({ + resolveKnowledgeConnectorMembersBinding: mocks.resolveMembersBinding, +})) + vi.mock('@/lib/knowledge/orchestration/connectors', () => ({ performCreateKnowledgeConnector: mocks.createConnector, performUpdateKnowledgeConnector: mocks.updateConnector, @@ -103,6 +108,7 @@ const crossWorkspaceContext = { const connectorContext = { ...crossWorkspaceContext, + access: { get: async () => ({ kind: 'workspace' as const, tokens: ['ws', 'pub'] as const }) }, connectorId: 'connector-b', connector: { id: 'connector-b', @@ -832,3 +838,69 @@ describe('knowledge connector application use cases', () => { }) }) }) + +describe('members-mode connector creation', () => { + const sessionPrincipal = { kind: 'session' as const, userId: 'user-1', sessionId: 'session-1' } + const membersInput = { + knowledgeBaseId: 'knowledge-b', + connectorType: 'google_drive', + sourceConfig: { folderId: ['f-1'] }, + syncIntervalMinutes: 1440, + accessMode: 'members' as const, + credentialGroupId: 'group-1', + credentialGroupOptionId: 'option-1', + } + + beforeEach(() => { + vi.clearAllMocks() + mocks.resolveKnowledgeBase.mockResolvedValue(crossWorkspaceContext) + mocks.getUserPermissionConfig.mockResolvedValue(DEFAULT_PERMISSION_GROUP_CONFIG) + mocks.resolveMembersBinding.mockResolvedValue({ + credentialGroupId: 'group-1', + credentialGroupOptionId: 'option-1', + workspaceId: 'workspace-b', + }) + mocks.createConnector.mockResolvedValue({ + success: true, + connector: { + id: 'connector-1', + connectorType: 'google_drive', + syncIntervalMinutes: 1440, + accessMode: 'members', + credentialId: null, + }, + }) + }) + + it('refuses members mode to a member below admin', async () => { + mocks.resolvePermission.mockResolvedValue('write') + + await expect( + createKnowledgeConnector.execute({ principal: sessionPrincipal, input: membersInput }) + ).rejects.toMatchObject({ name: 'InsufficientWorkspacePermissionsError' }) + expect(mocks.createConnector).not.toHaveBeenCalled() + }) + + it('validates the binding and passes it through for an admin', async () => { + mocks.resolvePermission.mockResolvedValue('admin') + + await createKnowledgeConnector.execute({ principal: sessionPrincipal, input: membersInput }) + + expect(mocks.resolveMembersBinding).toHaveBeenCalledWith( + expect.objectContaining({ + workspaceId: 'workspace-b', + binding: { credentialGroupId: 'group-1', credentialGroupOptionId: 'option-1' }, + sourceConfig: membersInput.sourceConfig, + }) + ) + expect(mocks.createConnector).toHaveBeenCalledWith( + expect.objectContaining({ + membersBinding: expect.objectContaining({ + credentialGroupId: 'group-1', + credentialGroupOptionId: 'option-1', + }), + credentialId: undefined, + }) + ) + }) +}) diff --git a/apps/sim/lib/knowledge/application/connectors.ts b/apps/sim/lib/knowledge/application/connectors.ts index 5e704149b80..729ce72b0dc 100644 --- a/apps/sim/lib/knowledge/application/connectors.ts +++ b/apps/sim/lib/knowledge/application/connectors.ts @@ -1,10 +1,19 @@ import { AuditAction, AuditResourceType } from '@sim/audit' +import type { Principal } from '@sim/auth/principal' import { resolvePrincipalSubjectUserId } from '@sim/auth/principal' import { db } from '@sim/db' -import { document, knowledgeConnector, knowledgeConnectorSyncLog } from '@sim/db/schema' -import { and, asc, count, desc, eq, inArray, isNull } from 'drizzle-orm' +import { + document, + knowledgeBase, + knowledgeConnector, + knowledgeConnectorMember, + knowledgeConnectorMemberSyncLog, + knowledgeConnectorSyncLog, +} from '@sim/db/schema' +import { and, asc, count, desc, eq, inArray, isNull, lt, or, sql } from 'drizzle-orm' import { decryptApiKey } from '@/lib/api-key/crypto' import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' +import { requireCurrentHumanRole } from '@/lib/core/application' import { OrchestrationError } from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' import { @@ -12,6 +21,9 @@ import { getCredentialActorContext, resolveCredentialTokenIdentity, } from '@/lib/credentials/access' +import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' +import { createKnowledgeAccessProvider } from '@/lib/knowledge/access/scope' +import type { KnowledgeAccessScope } from '@/lib/knowledge/access/types' import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' import { resolveKnowledgeAttributedUserId, @@ -21,13 +33,23 @@ import { type ActiveKnowledgeResourceBaseContext, resolveActiveKnowledgeConnectorContext, resolveActiveKnowledgeResourceContext, + resolveKnowledgeWorkspaceContext, } from '@/lib/knowledge/application/contexts' import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { + resolveViewerConnectorMemberships, + type ViewerConnectorMembership, +} from '@/lib/knowledge/connectors/member-provisioning' +import { MEMBER_OBSERVATION_STALE_AFTER_HOURS } from '@/lib/knowledge/connectors/sync-limits' import { DEFAULT_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE, MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_MUTATION_ITEMS, MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE, } from '@/lib/knowledge/constants' +import { + type ResolvedMembersBinding, + resolveKnowledgeConnectorMembersBinding, +} from '@/lib/knowledge/orchestration/connector-access' import { getKnowledgeConnector, type KnowledgeConnectorRow, @@ -41,9 +63,12 @@ import type { KnowledgeOperationSource, KnowledgeOrchestrationResult, } from '@/lib/knowledge/orchestration/shared' +import { isMemberSyncStatus } from '@/lib/knowledge/types' +import { credentialProviderMatchesService, type ServiceProviderIdentity } from '@/lib/oauth' import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' import { CAPABILITY_RULES, refuseCapability } from '@/lib/permission-groups/capabilities' import { resolvePermissionGroupConfig } from '@/lib/permission-groups/config-scope.server' +import { getConnectorMeta } from '@/connectors/registry' interface KnowledgeConnectorApplicationInput { assertedWorkspaceId?: string @@ -70,6 +95,10 @@ export interface CreateKnowledgeConnectorInput extends KnowledgeConnectorApplica apiKey?: string sourceConfig: Record syncIntervalMinutes: number + /** `members` crawls per Credential Group member; admin only. Defaults to `workspace`. */ + accessMode?: 'workspace' | 'members' + credentialGroupId?: string + credentialGroupOptionId?: string resolveBillingAttribution?(workspaceId: string): Promise } @@ -142,7 +171,7 @@ async function assertConnectorTypeAllowed( refuseCapability('knowledge.connectors') } -function requireSuccessfulOutcome( +export function requireSuccessfulOutcome( outcome: KnowledgeOrchestrationResult, fallback: string ): asserts outcome is { success: true } & T { @@ -161,7 +190,7 @@ function connectorTarget(context: ActiveKnowledgeResourceBaseContext) { } } -function requireConnectorWorkspaceId(context: ActiveKnowledgeResourceBaseContext): string { +export function requireConnectorWorkspaceId(context: ActiveKnowledgeResourceBaseContext): string { if (!context.workspaceId) { throw new OrchestrationError('conflict', 'Knowledge base is missing workspace billing context') } @@ -172,6 +201,7 @@ async function resolveAuthorizedConnectorCredentialIdentity(input: { credentialId: string workspaceId: string actingUserId: string + service?: ServiceProviderIdentity }) { const access = await getCredentialActorContext(input.credentialId, input.actingUserId) if ( @@ -184,14 +214,32 @@ async function resolveAuthorizedConnectorCredentialIdentity(input: { 'Credential is not available to you in this workspace. Ask a credential administrator to grant access or select another credential.' ) } + if ( + input.service && + (!access.credential.providerId || + !credentialProviderMatchesService(access.credential.providerId, input.service)) + ) { + throw new OrchestrationError( + 'validation', + 'Credential belongs to another service. Select a credential for the connector’s own provider.' + ) + } return resolveCredentialTokenIdentity(input.credentialId, input.workspaceId) } -async function resolveConnectorCredentialAccessToken(input: { +/** + * The access token a connector syncs with, once the caller may use the + * credential in this workspace. Pass `service` to also refuse a credential + * of another provider: creation validates the source config with the token, + * which catches that on its own, but a mode switch stores the credential + * without a call to the source. + */ +export async function resolveConnectorCredentialAccessToken(input: { credentialId: string workspaceId: string actingUserId: string requestId: string + service?: ServiceProviderIdentity }): Promise { const identity = await resolveAuthorizedConnectorCredentialIdentity(input) if (!identity) return null @@ -270,9 +318,14 @@ async function validateConnectorSourceConfig(input: { export const listKnowledgeConnectors = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.listConnectors, - resolveContext: ({ input }: { input: ListKnowledgeConnectorsInput }) => - resolveActiveKnowledgeResourceContext(input), - async execute({ input, context }) { + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: ListKnowledgeConnectorsInput + }) => resolveActiveKnowledgeResourceContext(input, principal), + async execute({ principal, input, context }) { const sortOrder = input.sortOrder === 'asc' ? asc : desc const sortColumn = input.sortBy === 'connectorType' @@ -298,8 +351,20 @@ export const listKnowledgeConnectors = defineAuthorizedKnowledgeUseCase({ : await orderedQuery.limit(input.limit + 1).offset(offset) const hasMore = input.limit !== undefined && rows.length > input.limit const page = input.limit === undefined ? rows : rows.slice(0, input.limit) + const viewerUserId = principal.kind === 'session' ? principal.userId : null + const memberships = + viewerUserId && context.workspaceId + ? await resolveViewerConnectorMemberships({ + userId: viewerUserId, + workspaceId: context.workspaceId, + connectors: page, + }) + : new Map() return { - connectors: page.map(({ encryptedApiKey: _encryptedApiKey, ...rest }) => rest), + connectors: page.map(({ encryptedApiKey: _encryptedApiKey, ...rest }) => ({ + ...rest, + viewerMembership: memberships.get(rest.id) ?? null, + })), hasMore, offset, limit: input.limit ?? page.length, @@ -307,28 +372,194 @@ export const listKnowledgeConnectors = defineAuthorizedKnowledgeUseCase({ }, }) +export interface ListWorkspaceMemberConnectorsInput { + workspaceId: string +} + +/** + * Every per-member connector in the workspace and where the viewer stands + * with each, so a surface outside the knowledge base — Sim Search — can ask + * them to connect. Only connectors the viewer could actually read documents + * from are listed: the knowledge base must be live and in the workspace. + */ +/** Live documents per connector that the viewer's tokens match, for the Search tab's counts. */ +async function countViewerDocuments( + connectorIds: readonly string[], + access: KnowledgeAccessScope +): Promise> { + if (connectorIds.length === 0) return new Map() + const rows = await db + .select({ connectorId: document.connectorId, count: sql`count(*)::int` }) + .from(document) + .where( + and( + inArray(document.connectorId, [...connectorIds]), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNull(document.deletedAt), + knowledgeAccessCondition(access) + ) + ) + .groupBy(document.connectorId) + return new Map(rows.flatMap((row) => (row.connectorId ? [[row.connectorId, row.count]] : []))) +} + +export const listWorkspaceMemberConnectors = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.listWorkspaceMemberConnectors, + resolveContext: ({ input }: { input: ListWorkspaceMemberConnectorsInput }) => + resolveKnowledgeWorkspaceContext(input), + async execute({ principal, context }) { + const viewerUserId = resolvePrincipalSubjectUserId(principal) + if (!viewerUserId) return { connectors: [] } + const rows = await db + .select({ + knowledgeBaseId: knowledgeConnector.knowledgeBaseId, + knowledgeBaseName: knowledgeBase.name, + id: knowledgeConnector.id, + connectorType: knowledgeConnector.connectorType, + accessMode: knowledgeConnector.accessMode, + memberSyncStatus: knowledgeConnector.memberSyncStatus, + credentialGroupId: knowledgeConnector.credentialGroupId, + credentialGroupOptionId: knowledgeConnector.credentialGroupOptionId, + }) + .from(knowledgeConnector) + .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) + .where( + and( + eq(knowledgeBase.workspaceId, context.workspaceId), + isNull(knowledgeBase.deletedAt), + eq(knowledgeConnector.accessMode, 'members'), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + .orderBy(asc(knowledgeBase.name), asc(knowledgeConnector.createdAt)) + const [memberships, documentCounts] = await Promise.all([ + resolveViewerConnectorMemberships({ + userId: viewerUserId, + workspaceId: context.workspaceId, + connectors: rows, + }), + countViewerDocuments( + rows.map((row) => row.id), + await createKnowledgeAccessProvider(principal, { workspaceId: context.workspaceId }).get() + ), + ]) + return { + connectors: rows.flatMap((row) => { + const viewerMembership = memberships.get(row.id) + if (!isMemberSyncStatus(row.memberSyncStatus)) { + throw new OrchestrationError( + 'conflict', + `Unexpected member sync status ${row.memberSyncStatus}` + ) + } + return viewerMembership + ? [ + { + knowledgeBaseId: row.knowledgeBaseId, + knowledgeBaseName: row.knowledgeBaseName, + connectorId: row.id, + connectorType: row.connectorType, + memberSyncStatus: row.memberSyncStatus, + viewerMembership, + viewerDocumentCount: documentCounts.get(row.id) ?? 0, + }, + ] + : [] + }), + } + }, +}) + export const readKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.readConnector, - resolveContext: ({ input }: { input: ReadKnowledgeConnectorInput }) => - resolveActiveKnowledgeConnectorContext(input), - async execute({ context }) { + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: ReadKnowledgeConnectorInput + }) => resolveActiveKnowledgeConnectorContext(input, principal), + async execute({ principal, context }) { const connector = await getKnowledgeConnector(context.knowledgeBaseId, context.connectorId) if (!connector) throw new OrchestrationError('not_found', 'Connector not found') - const syncLogs = await db - .select() - .from(knowledgeConnectorSyncLog) - .where(eq(knowledgeConnectorSyncLog.connectorId, context.connectorId)) - .orderBy(desc(knowledgeConnectorSyncLog.startedAt)) - .limit(10) + const [syncLogs, memberSyncLogs, members] = await Promise.all([ + db + .select() + .from(knowledgeConnectorSyncLog) + .where(eq(knowledgeConnectorSyncLog.connectorId, context.connectorId)) + .orderBy(desc(knowledgeConnectorSyncLog.startedAt)) + .limit(10), + db + .select() + .from(knowledgeConnectorMemberSyncLog) + .where(eq(knowledgeConnectorMemberSyncLog.connectorId, context.connectorId)) + .orderBy(desc(knowledgeConnectorMemberSyncLog.startedAt)) + .limit(10), + connector.accessMode === 'members' + ? summarizeConnectorMembers(context.connectorId, connector.syncIntervalMinutes) + : { active: 0, suspended: 0, stale: 0 }, + ]) const { encryptedApiKey: _encryptedApiKey, ...connectorData } = connector - return { connector: { ...connectorData, syncLogs } } + const viewerUserId = principal.kind === 'session' ? principal.userId : null + const memberships = + viewerUserId && context.workspaceId + ? await resolveViewerConnectorMemberships({ + userId: viewerUserId, + workspaceId: context.workspaceId, + connectors: [connector], + }) + : new Map() + return { + connector: { + ...connectorData, + viewerMembership: memberships.get(connector.id) ?? null, + syncLogs, + memberSyncLogs, + members, + }, + } }, }) +/** + * How many members a connector has in each state, for the settings surface. + * Stale mirrors the scheduler's sweep window: an active member whose last + * complete listing is older than `max(24 h, 2 × interval)`. + */ +async function summarizeConnectorMembers( + connectorId: string, + syncIntervalMinutes: number +): Promise<{ active: number; suspended: number; stale: number }> { + const staleWindowMs = Math.max( + MEMBER_OBSERVATION_STALE_AFTER_HOURS * 60 * 60 * 1000, + 2 * syncIntervalMinutes * 60 * 1000 + ) + const staleCutoff = new Date(Date.now() - staleWindowMs) + const [row] = await db + .select({ + active: sql`count(*) FILTER (WHERE ${knowledgeConnectorMember.status} = 'active')::int`, + suspended: sql`count(*) FILTER (WHERE ${knowledgeConnectorMember.status} <> 'active')::int`, + stale: sql`count(*) FILTER (WHERE ${knowledgeConnectorMember.status} = 'active' AND ${or( + isNull(knowledgeConnectorMember.lastCompleteListingAt), + lt(knowledgeConnectorMember.lastCompleteListingAt, staleCutoff) + )})::int`, + }) + .from(knowledgeConnectorMember) + .where(eq(knowledgeConnectorMember.connectorId, connectorId)) + return { active: row?.active ?? 0, suspended: row?.suspended ?? 0, stale: row?.stale ?? 0 } +} + export const createKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.createConnector, - resolveContext: ({ input }: { input: CreateKnowledgeConnectorInput }) => - resolveActiveKnowledgeResourceContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: CreateKnowledgeConnectorInput + }) => resolveActiveKnowledgeResourceContext(input, principal), async execute({ principal, input, context, request }) { const requestId = generateRequestId() const workspaceId = requireConnectorWorkspaceId(context) @@ -339,13 +570,53 @@ export const createKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ workspaceId, input.connectorType ) + let membersBinding: ResolvedMembersBinding | undefined + if (input.accessMode === 'members') { + /** + * Members mode grants the connector every enrolled member's credential, + * which is an admin decision even though creating a connector is not. + */ + const subjectUserId = resolvePrincipalSubjectUserId(principal) + if (context.workspaceId === undefined) { + throw new OrchestrationError( + 'validation', + 'Per-member access needs a workspace knowledge base' + ) + } + if (!subjectUserId) { + throw new OrchestrationError( + 'forbidden', + 'A members-mode connector needs a signed-in admin' + ) + } + await requireCurrentHumanRole(subjectUserId, context, 'admin') + const connectorMeta = getConnectorMeta(input.connectorType) + if (!connectorMeta) { + throw new OrchestrationError('validation', `Unknown connector type: ${input.connectorType}`) + } + membersBinding = await resolveKnowledgeConnectorMembersBinding({ + workspaceId, + connectorMeta, + binding: + input.credentialGroupId && input.credentialGroupOptionId + ? { + credentialGroupId: input.credentialGroupId, + credentialGroupOptionId: input.credentialGroupOptionId, + } + : null, + actingUserId: subjectUserId, + sourceConfig: input.sourceConfig, + }) + } const outcome = await performCreateKnowledgeConnector({ knowledgeBase: connectorTarget(context), connectorType: input.connectorType, credentialId: input.credentialId, apiKey: input.apiKey, - sourceConfig: input.sourceConfig, + /** Members mode stores the config with its listing caps cleared. */ + sourceConfig: membersBinding?.sourceConfig ?? input.sourceConfig, syncIntervalMinutes: input.syncIntervalMinutes, + membersBinding, resolveBillingAttribution: () => input.resolveBillingAttribution?.(workspaceId) ?? resolveKnowledgeBillingAttribution(principal, context), @@ -379,6 +650,7 @@ export const createKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ connectorType: result.connector.connectorType, syncIntervalMinutes: result.connector.syncIntervalMinutes, authMode: result.connector.credentialId ? 'oauth' : 'apiKey', + accessMode: result.connector.accessMode, }, }), }) @@ -392,8 +664,13 @@ export const createKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ */ export const updateKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.updateConnector, - resolveContext: ({ input }: { input: UpdateKnowledgeConnectorInput }) => - resolveActiveKnowledgeConnectorContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: UpdateKnowledgeConnectorInput + }) => resolveActiveKnowledgeConnectorContext(input, principal), async execute({ principal, input, context, request }) { const requestId = generateRequestId() const actingUserId = resolveKnowledgeAttributedUserId(principal, context) @@ -451,8 +728,13 @@ export const updateKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ export const deleteKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.deleteConnector, - resolveContext: ({ input }: { input: DeleteKnowledgeConnectorInput }) => - resolveActiveKnowledgeConnectorContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: DeleteKnowledgeConnectorInput + }) => resolveActiveKnowledgeConnectorContext(input, principal), async execute({ principal, input, context, request }) { const outcome = await performDeleteKnowledgeConnector({ knowledgeBase: connectorTarget(context), @@ -510,8 +792,13 @@ export const deleteKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ */ export const syncKnowledgeConnector = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.syncConnector, - resolveContext: ({ input }: { input: SyncKnowledgeConnectorInput }) => - resolveActiveKnowledgeConnectorContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: SyncKnowledgeConnectorInput + }) => resolveActiveKnowledgeConnectorContext(input, principal), async execute({ principal, input, context, request }) { const workspaceId = requireConnectorWorkspaceId(context) // permission-group-enforced: knowledge.connectors — needs the persisted connector type, which the funnel never sees @@ -572,8 +859,13 @@ const connectorDocumentSelection = { export const listKnowledgeConnectorDocuments = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.listConnectorDocuments, - resolveContext: ({ input }: { input: ListKnowledgeConnectorDocumentsInput }) => - resolveActiveKnowledgeConnectorContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: ListKnowledgeConnectorDocumentsInput + }) => resolveActiveKnowledgeConnectorContext(input, principal), async execute({ input, context }) { const limit = input.limit ?? DEFAULT_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE const offset = input.offset ?? 0 @@ -597,6 +889,7 @@ export const listKnowledgeConnectorDocuments = defineAuthorizedKnowledgeUseCase( eq(document.connectorId, context.connectorId), isNull(document.archivedAt), isNull(document.deletedAt), + knowledgeAccessCondition(await context.access.get()), ] as const const [[activeCount], excludedCountRows] = await Promise.all([ db @@ -634,8 +927,13 @@ export const listKnowledgeConnectorDocuments = defineAuthorizedKnowledgeUseCase( export const updateKnowledgeConnectorDocuments = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.updateConnectorDocuments, - resolveContext: ({ input }: { input: UpdateKnowledgeConnectorDocumentsInput }) => - resolveActiveKnowledgeConnectorContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: UpdateKnowledgeConnectorDocumentsInput + }) => resolveActiveKnowledgeConnectorContext(input, principal), async execute({ input, context }) { if (input.documentIds.length === 0) { throw new OrchestrationError('validation', 'At least one connector document is required') @@ -657,7 +955,8 @@ export const updateKnowledgeConnectorDocuments = defineAuthorizedKnowledgeUseCas inArray(document.id, documentIds), eq(document.userExcluded, restoring), isNull(document.archivedAt), - isNull(document.deletedAt) + isNull(document.deletedAt), + knowledgeAccessCondition(await context.access.get()) ) ) .returning({ id: document.id }) diff --git a/apps/sim/lib/knowledge/application/contexts.test.ts b/apps/sim/lib/knowledge/application/contexts.test.ts index 7e3a30440fd..37c9b444f39 100644 --- a/apps/sim/lib/knowledge/application/contexts.test.ts +++ b/apps/sim/lib/knowledge/application/contexts.test.ts @@ -12,6 +12,13 @@ const mocks = vi.hoisted(() => ({ getConnector: vi.fn(), loadWorkspace: vi.fn(), loadWorkspaceIncludingArchived: vi.fn(), + createAccessProvider: vi.fn(() => ({ + get: async () => ({ kind: 'workspace', tokens: ['pub', 'ws'] }), + })), +})) + +vi.mock('@/lib/knowledge/access/scope', () => ({ + createKnowledgeAccessProvider: mocks.createAccessProvider, })) vi.mock('@/lib/knowledge/service', () => ({ @@ -53,6 +60,7 @@ const workspace = { billedAccountUserId: 'billing-user-1', } const knowledgeBase = { id: 'knowledge-1', workspaceId: 'workspace-1' } +const principal = { kind: 'session' as const, userId: 'user-1', sessionId: 'session-1' } describe('knowledge application contexts', () => { beforeEach(() => { @@ -82,7 +90,7 @@ describe('knowledge application contexts', () => { mocks.loadWorkspace.mockResolvedValueOnce(null) await expect( - resolveActiveKnowledgeBaseContext({ knowledgeBaseId: 'knowledge-1' }) + resolveActiveKnowledgeBaseContext({ knowledgeBaseId: 'knowledge-1' }, principal) ).rejects.toMatchObject({ code: 'not_found', message: 'Knowledge base not found' }) }) @@ -91,7 +99,7 @@ describe('knowledge application contexts', () => { mocks.loadWorkspace.mockRejectedValueOnce(failure) await expect( - resolveActiveKnowledgeBaseContext({ knowledgeBaseId: 'knowledge-1' }) + resolveActiveKnowledgeBaseContext({ knowledgeBaseId: 'knowledge-1' }, principal) ).rejects.toBe(failure) }) @@ -103,7 +111,7 @@ describe('knowledge application contexts', () => { }) await expect( - resolveActiveKnowledgeResourceContext({ knowledgeBaseId: 'legacy-knowledge' }) + resolveActiveKnowledgeResourceContext({ knowledgeBaseId: 'legacy-knowledge' }, principal) ).resolves.toMatchObject({ knowledgeBaseId: 'legacy-knowledge', workspaceId: undefined, @@ -120,10 +128,13 @@ describe('knowledge application contexts', () => { }) await expect( - resolveActiveKnowledgeResourceContext({ - knowledgeBaseId: 'legacy-knowledge', - assertedWorkspaceId: 'workspace-1', - }) + resolveActiveKnowledgeResourceContext( + { + knowledgeBaseId: 'legacy-knowledge', + assertedWorkspaceId: 'workspace-1', + }, + principal + ) ).rejects.toMatchObject({ code: 'not_found' }) }) @@ -139,10 +150,13 @@ describe('knowledge application contexts', () => { }) await expect( - resolveCanonicalActiveKnowledgeDocumentContext({ - knowledgeBaseId: 'legacy-knowledge', - documentId: 'legacy-document', - }) + resolveCanonicalActiveKnowledgeDocumentContext( + { + knowledgeBaseId: 'legacy-knowledge', + documentId: 'legacy-document', + }, + principal + ) ).resolves.toMatchObject({ documentId: 'legacy-document', knowledgeBaseId: 'legacy-knowledge', @@ -174,25 +188,31 @@ describe('knowledge application contexts', () => { }) }) - it('resolves a document parent canonically before comparing the trusted workspace', async () => { + it('resolves the asserted parent and conceals a foreign document without loading it', async () => { await expect( - resolveCanonicalActiveKnowledgeDocumentContext({ - knowledgeBaseId: 'knowledge-b', - documentId: 'document-b', - assertedWorkspaceId: 'workspace-a', - }) + resolveCanonicalActiveKnowledgeDocumentContext( + { + knowledgeBaseId: 'knowledge-b', + documentId: 'document-b', + assertedWorkspaceId: 'workspace-a', + }, + principal + ) ).rejects.toMatchObject({ code: 'not_found' }) - expect(mocks.getDocumentById).toHaveBeenCalledWith('document-b') expect(mocks.getKnowledgeBase).toHaveBeenCalledWith('knowledge-b') + expect(mocks.getDocumentById).not.toHaveBeenCalled() }) it('resolves a tag parent canonically before comparing the trusted workspace', async () => { await expect( - resolveActiveKnowledgeTagContext({ - tagDefinitionId: 'tag-b', - assertedWorkspaceId: 'workspace-a', - }) + resolveActiveKnowledgeTagContext( + { + tagDefinitionId: 'tag-b', + assertedWorkspaceId: 'workspace-a', + }, + principal + ) ).rejects.toMatchObject({ code: 'not_found' }) expect(mocks.getTag).toHaveBeenCalledWith('tag-b') @@ -201,10 +221,13 @@ describe('knowledge application contexts', () => { it('resolves a connector parent canonically before comparing the trusted workspace', async () => { await expect( - resolveActiveKnowledgeConnectorContext({ - connectorId: 'connector-b', - assertedWorkspaceId: 'workspace-a', - }) + resolveActiveKnowledgeConnectorContext( + { + connectorId: 'connector-b', + assertedWorkspaceId: 'workspace-a', + }, + principal + ) ).rejects.toMatchObject({ code: 'not_found' }) expect(mocks.getConnector).toHaveBeenCalledWith('connector-b') diff --git a/apps/sim/lib/knowledge/application/contexts.ts b/apps/sim/lib/knowledge/application/contexts.ts index a6df12a0dc4..c707b630933 100644 --- a/apps/sim/lib/knowledge/application/contexts.ts +++ b/apps/sim/lib/knowledge/application/contexts.ts @@ -1,7 +1,10 @@ +import type { Principal } from '@sim/auth/principal' import { db } from '@sim/db' import { embedding } from '@sim/db/schema' import { and, eq } from 'drizzle-orm' import { OrchestrationError } from '@/lib/core/orchestration/types' +import { createKnowledgeAccessProvider } from '@/lib/knowledge/access/scope' +import type { KnowledgeAccessProvider } from '@/lib/knowledge/access/types' import type { KnowledgeAuthorizationContext, LegacyPersonalKnowledgeAuthorizationContext, @@ -35,16 +38,30 @@ export interface LegacyPersonalKnowledgeContext export type KnowledgeResourceContext = KnowledgeWorkspaceContext | LegacyPersonalKnowledgeContext -export interface ActiveKnowledgeBaseContext extends KnowledgeWorkspaceContext { - knowledgeBaseId: string - knowledgeBase: KnowledgeBaseWithCounts +/** + * What the calling principal may read within this knowledge base, resolved + * lazily so a write-only operation never pays for it. Every document loader + * requires the resolved scope; the resolvers below await it before loading a + * document, so a document the caller may not read is reported as absent from + * the very first read. + */ +interface KnowledgeAccessBearingContext { + access: KnowledgeAccessProvider } -export type ActiveKnowledgeResourceBaseContext = KnowledgeResourceContext & { +export interface ActiveKnowledgeBaseContext + extends KnowledgeWorkspaceContext, + KnowledgeAccessBearingContext { knowledgeBaseId: string knowledgeBase: KnowledgeBaseWithCounts } +export type ActiveKnowledgeResourceBaseContext = KnowledgeResourceContext & + KnowledgeAccessBearingContext & { + knowledgeBaseId: string + knowledgeBase: KnowledgeBaseWithCounts + } + export type ActiveKnowledgeDocumentContext = ActiveKnowledgeResourceBaseContext & { documentId: string document: ActiveKnowledgeDocument @@ -116,10 +133,13 @@ async function requireKnowledgeBase(knowledgeBaseId: string, workspaceId: string return knowledgeBase as typeof knowledgeBase & { workspaceId: string } } -export async function resolveActiveKnowledgeBaseContext(input: { - knowledgeBaseId: string - assertedWorkspaceId?: string -}): Promise { +export async function resolveActiveKnowledgeBaseContext( + input: { + knowledgeBaseId: string + assertedWorkspaceId?: string + }, + principal: Principal +): Promise { const knowledgeBase = await requireKnowledgeBase(input.knowledgeBaseId, input.assertedWorkspaceId) const workspaceContext = await loadKnowledgeWorkspaceContext(knowledgeBase.workspaceId) if (!workspaceContext) throw new OrchestrationError('not_found', 'Knowledge base not found') @@ -127,6 +147,7 @@ export async function resolveActiveKnowledgeBaseContext(input: { ...workspaceContext, knowledgeBaseId: knowledgeBase.id, knowledgeBase, + access: createKnowledgeAccessProvider(principal, { workspaceId: knowledgeBase.workspaceId }), } } @@ -139,10 +160,16 @@ export async function resolveActiveKnowledgeBaseContext(input: { */ export async function resolveActiveKnowledgeBaseInWorkspace( knowledgeBaseId: string, - workspaceContext: KnowledgeWorkspaceContext + workspaceContext: KnowledgeWorkspaceContext, + principal: Principal ): Promise { const knowledgeBase = await requireKnowledgeBase(knowledgeBaseId, workspaceContext.workspaceId) - return { ...workspaceContext, knowledgeBaseId: knowledgeBase.id, knowledgeBase } + return { + ...workspaceContext, + knowledgeBaseId: knowledgeBase.id, + knowledgeBase, + access: createKnowledgeAccessProvider(principal, { workspaceId: workspaceContext.workspaceId }), + } } /** @@ -181,10 +208,13 @@ export async function resolveArchivedKnowledgeBaseContext(input: { } } -export async function resolveActiveKnowledgeResourceContext(input: { - knowledgeBaseId: string - assertedWorkspaceId?: string -}): Promise { +export async function resolveActiveKnowledgeResourceContext( + input: { + knowledgeBaseId: string + assertedWorkspaceId?: string + }, + principal: Principal +): Promise { const knowledgeBase = await getKnowledgeBaseById(input.knowledgeBaseId) if ( !knowledgeBase || @@ -199,6 +229,7 @@ export async function resolveActiveKnowledgeResourceContext(input: { legacyPersonalOwnerUserId: knowledgeBase.userId, knowledgeBaseId: knowledgeBase.id, knowledgeBase, + access: createKnowledgeAccessProvider(principal, {}), } } const workspaceContext = await loadKnowledgeWorkspaceContext(knowledgeBase.workspaceId) @@ -207,16 +238,24 @@ export async function resolveActiveKnowledgeResourceContext(input: { ...workspaceContext, knowledgeBaseId: knowledgeBase.id, knowledgeBase, + access: createKnowledgeAccessProvider(principal, { workspaceId: knowledgeBase.workspaceId }), } } -export async function resolveActiveKnowledgeDocumentContext(input: { - knowledgeBaseId: string - documentId: string - assertedWorkspaceId?: string -}): Promise { - const context = await resolveActiveKnowledgeResourceContext(input) - const document = await getKnowledgeDocument(context.knowledgeBaseId, input.documentId) +export async function resolveActiveKnowledgeDocumentContext( + input: { + knowledgeBaseId: string + documentId: string + assertedWorkspaceId?: string + }, + principal: Principal +): Promise { + const context = await resolveActiveKnowledgeResourceContext(input, principal) + const document = await getKnowledgeDocument( + context.knowledgeBaseId, + input.documentId, + await context.access.get() + ) if (!document) throw new OrchestrationError('not_found', 'Document not found') return { ...context, @@ -225,19 +264,26 @@ export async function resolveActiveKnowledgeDocumentContext(input: { } } -export async function resolveCanonicalActiveKnowledgeDocumentContext(input: { - knowledgeBaseId: string - documentId: string - assertedWorkspaceId?: string -}): Promise { - const document = await getKnowledgeDocumentById(input.documentId) - if (!document || document.knowledgeBaseId !== input.knowledgeBaseId) { +/** + * Resolves a document by its canonical id and only then trusts the asserted + * parent. The access scope needs the workspace, which is only known once the + * asserted knowledge base is loaded, so the base is resolved first and the + * document is then required to belong to it — a mismatch is concealed as + * not-found exactly as before. + */ +export async function resolveCanonicalActiveKnowledgeDocumentContext( + input: { + knowledgeBaseId: string + documentId: string + assertedWorkspaceId?: string + }, + principal: Principal +): Promise { + const context = await resolveActiveKnowledgeResourceContext(input, principal) + const document = await getKnowledgeDocumentById(input.documentId, await context.access.get()) + if (!document || document.knowledgeBaseId !== context.knowledgeBaseId) { throw new OrchestrationError('not_found', 'Document not found') } - const context = await resolveActiveKnowledgeResourceContext({ - knowledgeBaseId: document.knowledgeBaseId, - assertedWorkspaceId: input.assertedWorkspaceId, - }) return { ...context, documentId: document.id, @@ -245,12 +291,15 @@ export async function resolveCanonicalActiveKnowledgeDocumentContext(input: { } } -export async function resolveActiveKnowledgeChunkContext(input: { - knowledgeBaseId: string - documentId: string - chunkId: string - assertedWorkspaceId?: string -}): Promise { +export async function resolveActiveKnowledgeChunkContext( + input: { + knowledgeBaseId: string + documentId: string + chunkId: string + assertedWorkspaceId?: string + }, + principal: Principal +): Promise { const [chunk] = await db .select() .from(embedding) @@ -259,7 +308,7 @@ export async function resolveActiveKnowledgeChunkContext(input: { if (!chunk || chunk.knowledgeBaseId !== input.knowledgeBaseId) { throw new OrchestrationError('not_found', 'Chunk not found') } - const context = await resolveCanonicalActiveKnowledgeDocumentContext(input) + const context = await resolveCanonicalActiveKnowledgeDocumentContext(input, principal) return { ...context, chunkId: chunk.id, @@ -267,11 +316,14 @@ export async function resolveActiveKnowledgeChunkContext(input: { } } -export async function resolveActiveKnowledgeTagContext(input: { - tagDefinitionId: string - knowledgeBaseId?: string - assertedWorkspaceId?: string -}): Promise { +export async function resolveActiveKnowledgeTagContext( + input: { + tagDefinitionId: string + knowledgeBaseId?: string + assertedWorkspaceId?: string + }, + principal: Principal +): Promise { const tagDefinition = await getTagDefinitionById(input.tagDefinitionId) if ( !tagDefinition || @@ -279,10 +331,13 @@ export async function resolveActiveKnowledgeTagContext(input: { ) { throw new OrchestrationError('not_found', 'Tag definition not found') } - const context = await resolveActiveKnowledgeResourceContext({ - knowledgeBaseId: tagDefinition.knowledgeBaseId, - assertedWorkspaceId: input.assertedWorkspaceId, - }) + const context = await resolveActiveKnowledgeResourceContext( + { + knowledgeBaseId: tagDefinition.knowledgeBaseId, + assertedWorkspaceId: input.assertedWorkspaceId, + }, + principal + ) return { ...context, tagDefinitionId: tagDefinition.id, @@ -290,11 +345,14 @@ export async function resolveActiveKnowledgeTagContext(input: { } } -export async function resolveActiveKnowledgeConnectorContext(input: { - connectorId: string - knowledgeBaseId?: string - assertedWorkspaceId?: string -}): Promise { +export async function resolveActiveKnowledgeConnectorContext( + input: { + connectorId: string + knowledgeBaseId?: string + assertedWorkspaceId?: string + }, + principal: Principal +): Promise { const connector = await getActiveKnowledgeConnectorReference(input.connectorId) if ( !connector || @@ -302,10 +360,13 @@ export async function resolveActiveKnowledgeConnectorContext(input: { ) { throw new OrchestrationError('not_found', 'Connector not found') } - const context = await resolveActiveKnowledgeResourceContext({ - knowledgeBaseId: connector.knowledgeBaseId, - assertedWorkspaceId: input.assertedWorkspaceId, - }) + const context = await resolveActiveKnowledgeResourceContext( + { + knowledgeBaseId: connector.knowledgeBaseId, + assertedWorkspaceId: input.assertedWorkspaceId, + }, + principal + ) return { ...context, connectorId: connector.id, diff --git a/apps/sim/lib/knowledge/application/documents.test.ts b/apps/sim/lib/knowledge/application/documents.test.ts index 2fed1fc8d9a..e3fad4463aa 100644 --- a/apps/sim/lib/knowledge/application/documents.test.ts +++ b/apps/sim/lib/knowledge/application/documents.test.ts @@ -2,7 +2,7 @@ * @vitest-environment node */ -import { dbChainMockFns, resetDbChainMock } from '@sim/testing' +import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ @@ -103,6 +103,7 @@ vi.mock('@/lib/uploads/server/metadata', () => ({ vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: mocks.captureServerEvent })) import { OrchestrationError } from '@/lib/core/orchestration/types' +import { WORKSPACE_ACCESS_SCOPE } from '@/lib/knowledge/access/scope' import { bulkDeleteKnowledgeDocuments, createKnowledgeDocuments, @@ -113,7 +114,11 @@ import { upsertKnowledgeDocument, } from '@/lib/knowledge/application/documents' +/** Every mocked context carries the workspace read scope the resolvers would attach. */ +const knowledgeAccess = { get: async () => WORKSPACE_ACCESS_SCOPE } + const context = { + access: knowledgeAccess, workspaceId: 'workspace-1', workspaceOrganizationId: null, allowPersonalApiKeys: true, @@ -251,6 +256,69 @@ describe('knowledge document application use cases', () => { }) }) + /** + * The document being replaced is looked up and deleted under the caller's + * access, so a restricted document is neither confirmed nor replaced, and one + * that leaves the caller's reach mid-request keeps the replacement as an + * ordinary upload. + */ + it('replaces only a document the caller may read, under the same access', async () => { + queueTableRows(schemaMock.document, [{ id: 'existing-1' }]) + + const result = await upsertKnowledgeDocument.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { + knowledgeBaseId: 'knowledge-1', + assertedWorkspaceId: 'workspace-1', + filename: document.filename, + fileUrl: document.fileUrl, + fileSize: document.fileSize, + mimeType: document.mimeType, + resolveBillingAttribution: async () => ({ + actorUserId: 'user-1', + workspaceId: 'workspace-1', + }), + resolveSecretProvenances: () => undefined, + }, + }) + + expect(result).toMatchObject({ isUpdate: true, previousDocumentId: 'existing-1' }) + expect(mocks.deleteDocument).toHaveBeenCalledWith( + 'knowledge-1', + 'existing-1', + expect.any(String), + WORKSPACE_ACCESS_SCOPE + ) + expect(mocks.deleteDocumentById).not.toHaveBeenCalled() + }) + + it('keeps the replacement when the previous document left the caller’s reach', async () => { + queueTableRows(schemaMock.document, [{ id: 'existing-1' }]) + mocks.deleteDocument.mockRejectedValueOnce( + new OrchestrationError('not_found', 'Document not found') + ) + + const result = await upsertKnowledgeDocument.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { + knowledgeBaseId: 'knowledge-1', + assertedWorkspaceId: 'workspace-1', + filename: document.filename, + fileUrl: document.fileUrl, + fileSize: document.fileSize, + mimeType: document.mimeType, + resolveBillingAttribution: async () => ({ + actorUserId: 'user-1', + workspaceId: 'workspace-1', + }), + resolveSecretProvenances: () => undefined, + }, + }) + + expect(result).toMatchObject({ isUpdate: false, previousDocumentId: null }) + expect(mocks.deleteDocumentById).not.toHaveBeenCalled() + }) + it('authorizes the canonical knowledge base before listing documents', async () => { await listKnowledgeDocuments.execute({ principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, @@ -262,7 +330,8 @@ describe('knowledge document application use cases', () => { }) expect(mocks.resolveKnowledgeBase).toHaveBeenCalledWith( - expect.objectContaining({ assertedWorkspaceId: 'workspace-1' }) + expect.objectContaining({ assertedWorkspaceId: 'workspace-1' }), + expect.objectContaining({ kind: 'session' }) ) expect(mocks.resolvePermission.mock.invocationCallOrder[0]).toBeLessThan( mocks.getDocuments.mock.invocationCallOrder[0] @@ -271,6 +340,7 @@ describe('knowledge document application use cases', () => { it('lets the owner list documents in a legacy personal knowledge base', async () => { mocks.resolveKnowledgeBase.mockResolvedValueOnce({ + access: knowledgeAccess, workspaceId: undefined, legacyPersonalOwnerUserId: 'user-1', knowledgeBaseId: 'legacy-knowledge', @@ -288,13 +358,15 @@ describe('knowledge document application use cases', () => { expect(mocks.getDocuments).toHaveBeenCalledWith( 'legacy-knowledge', expect.any(Object), - expect.any(String) + expect.any(String), + WORKSPACE_ACCESS_SCOPE ) expect(mocks.recordAudit).not.toHaveBeenCalled() }) it('projects mutation audit entries for an owning legacy personal principal', async () => { mocks.resolveDocument.mockResolvedValueOnce({ + access: knowledgeAccess, workspaceId: undefined, legacyPersonalOwnerUserId: 'user-1', knowledgeBaseId: 'legacy-knowledge', @@ -326,6 +398,7 @@ describe('knowledge document application use cases', () => { it('conceals legacy personal documents from a non-owner', async () => { mocks.resolveKnowledgeBase.mockResolvedValueOnce({ + access: knowledgeAccess, workspaceId: undefined, legacyPersonalOwnerUserId: 'user-1', knowledgeBaseId: 'legacy-knowledge', @@ -511,7 +584,8 @@ describe('knowledge document application use cases', () => { expect(mocks.deleteDocument).toHaveBeenCalledWith( 'knowledge-1', 'document-1', - expect.any(String) + expect.any(String), + WORKSPACE_ACCESS_SCOPE ) expect(mocks.recordAudit).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/apps/sim/lib/knowledge/application/documents.ts b/apps/sim/lib/knowledge/application/documents.ts index 1cefe262fe2..c1a6961b4e3 100644 --- a/apps/sim/lib/knowledge/application/documents.ts +++ b/apps/sim/lib/knowledge/application/documents.ts @@ -1,4 +1,5 @@ import { AuditAction, AuditResourceType } from '@sim/audit' +import type { Principal } from '@sim/auth/principal' import { db } from '@sim/db' import { document as documentTable } from '@sim/db/schema' import { createLogger } from '@sim/logger' @@ -10,6 +11,7 @@ import { import { authorizeWorkspaceOperation } from '@/lib/core/application' import { asOrchestrationError, OrchestrationError } from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' +import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' import { knowledgeDelegationPolicy } from '@/lib/knowledge/application/authorization' import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' import { @@ -285,8 +287,13 @@ export interface UpsertKnowledgeDocumentInput extends UploadKnowledgeDocumentAdm */ export const listKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.listDocuments, - resolveContext: ({ input }: { input: ListKnowledgeDocumentsInput }) => - resolveActiveKnowledgeResourceContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: ListKnowledgeDocumentsInput + }) => resolveActiveKnowledgeResourceContext(input, principal), async execute({ input, context }) { const limit = input.limit ?? 50 const offset = input.offset ?? 0 @@ -316,7 +323,8 @@ export const listKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ sortOrder: input.sortOrder, tagFilters: tagFilters.length > 0 ? tagFilters : undefined, }, - generateRequestId() + generateRequestId(), + await context.access.get() ) return { ...result, @@ -330,8 +338,13 @@ export const listKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ export const readKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.readDocument, - resolveContext: ({ input }: { input: ReadKnowledgeDocumentInput }) => - resolveActiveKnowledgeDocumentContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: ReadKnowledgeDocumentInput + }) => resolveActiveKnowledgeDocumentContext(input, principal), async execute({ context }: { context: ActiveKnowledgeDocumentContext }) { return { document: context.document, @@ -343,8 +356,13 @@ export const readKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ export const admitKnowledgeDocumentUpload = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.uploadDocument, - resolveContext: ({ input }: { input: UploadKnowledgeDocumentAdmissionInput }) => - resolveActiveKnowledgeBaseContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: UploadKnowledgeDocumentAdmissionInput + }) => resolveActiveKnowledgeBaseContext(input, principal), async execute({ principal, context }) { const billingAttribution = await resolveKnowledgeBillingAttribution(principal, context) const usage = await checkAttributedUsageLimits(billingAttribution) @@ -363,8 +381,13 @@ export const admitKnowledgeDocumentUpload = defineAuthorizedKnowledgeUseCase({ export const uploadKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.uploadDocument, - resolveContext: ({ input }: { input: UploadKnowledgeDocumentInput }) => - resolveActiveKnowledgeBaseContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: UploadKnowledgeDocumentInput + }) => resolveActiveKnowledgeBaseContext(input, principal), async execute({ principal, input, context }) { if (input.file.fileSize < 0 || input.file.fileSize > MAX_KNOWLEDGE_DOCUMENT_FILE_SIZE) { throw new OrchestrationError( @@ -416,7 +439,7 @@ export const uploadKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ throw new Error('Knowledge document storage returned a path with an unexpected query') } - const registrationContext = await resolveActiveKnowledgeBaseContext(input) + const registrationContext = await resolveActiveKnowledgeBaseContext(input, principal) await authorizeWorkspaceOperation( principal, knowledgeOperations.uploadDocument, @@ -476,8 +499,13 @@ export const uploadKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ export const createKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.uploadDocument, - resolveContext: ({ input }: { input: CreateKnowledgeDocumentsInput }) => - resolveActiveKnowledgeResourceContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: CreateKnowledgeDocumentsInput + }) => resolveActiveKnowledgeResourceContext(input, principal), async execute({ principal, input, context, request }) { if (input.documents.length === 0) { throw new OrchestrationError('validation', 'No documents specified') @@ -611,8 +639,13 @@ export const createKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ export const upsertKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.uploadDocument, - resolveContext: ({ input }: { input: UpsertKnowledgeDocumentInput }) => - resolveActiveKnowledgeResourceContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: UpsertKnowledgeDocumentInput + }) => resolveActiveKnowledgeResourceContext(input, principal), async execute({ principal, input, context }) { const { billingAttribution, usage, userId } = await resolveKnowledgeUsageAdmission( principal, @@ -628,6 +661,11 @@ export const upsertKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ userId, workspaceId: context.workspaceId, }) + /** + * Only a document the caller may read counts as the one being replaced: + * a restricted document is neither confirmed to exist nor replaced. + */ + const access = await context.access.get() let existingDocumentId: string | null = null if (input.documentId) { const [existing] = await db @@ -637,7 +675,8 @@ export const upsertKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ and( eq(documentTable.id, input.documentId), eq(documentTable.knowledgeBaseId, context.knowledgeBaseId), - isNull(documentTable.deletedAt) + isNull(documentTable.deletedAt), + knowledgeAccessCondition(access) ) ) .limit(1) @@ -650,7 +689,8 @@ export const upsertKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ and( eq(documentTable.filename, input.filename), eq(documentTable.knowledgeBaseId, context.knowledgeBaseId), - isNull(documentTable.deletedAt) + isNull(documentTable.deletedAt), + knowledgeAccessCondition(access) ) ) .limit(1) @@ -676,18 +716,36 @@ export const upsertKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ if (!createdDocument) throw new Error('Knowledge document upsert created no document record') if (existingDocumentId) { try { - await deleteDocument(existingDocumentId, requestId) + await deleteKnowledgeDocumentInKnowledgeBase( + context.knowledgeBaseId, + existingDocumentId, + requestId, + access + ) } catch (error) { - try { - await deleteDocument(createdDocument.documentId, requestId) - } catch (rollbackError) { - logger.error('Failed to remove replacement after document upsert failure', { + /** + * The previous document went away — or out of the caller's reach — + * between the lookup and the delete. The replacement is an ordinary + * upload the caller may make, so it stays. + */ + if (error instanceof OrchestrationError && error.code === 'not_found') { + logger.warn('Document being replaced was no longer visible; keeping the replacement', { knowledgeBaseId: context.knowledgeBaseId, - documentId: createdDocument.documentId, - rollbackError, + previousDocumentId: existingDocumentId, }) + existingDocumentId = null + } else { + try { + await deleteDocument(createdDocument.documentId, requestId) + } catch (rollbackError) { + logger.error('Failed to remove replacement after document upsert failure', { + knowledgeBaseId: context.knowledgeBaseId, + documentId: createdDocument.documentId, + rollbackError, + }) + } + throw new Error('Failed to replace existing document', { cause: error }) } - throw new Error('Failed to replace existing document', { cause: error }) } } void dispatchDocumentProcessing({ @@ -731,13 +789,19 @@ export const upsertKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ export const deleteKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.deleteDocument, - resolveContext: ({ input }: { input: DeleteKnowledgeDocumentInput }) => - resolveActiveKnowledgeDocumentContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: DeleteKnowledgeDocumentInput + }) => resolveActiveKnowledgeDocumentContext(input, principal), async execute({ context }: { context: ActiveKnowledgeDocumentContext }) { await deleteKnowledgeDocumentInKnowledgeBase( context.knowledgeBaseId, context.documentId, - generateRequestId() + generateRequestId(), + await context.access.get() ) return { id: context.documentId, @@ -768,8 +832,10 @@ export const deleteKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ export const bulkDeleteKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.bulkDeleteDocuments, async resolveContext({ + principal, input, }: { + principal: Principal input: BulkDeleteKnowledgeDocumentsInput }): Promise { const documentIds = requireBoundedKnowledgeBatch( @@ -778,7 +844,7 @@ export const bulkDeleteKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ BULK_DELETE_KNOWLEDGE_DOCUMENTS_COST_POLICY.maxItems ) return { - ...(await resolveActiveKnowledgeResourceContext(input)), + ...(await resolveActiveKnowledgeResourceContext(input, principal)), documentIds, } }, @@ -794,11 +860,14 @@ export const bulkDeleteKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ for (const documentId of context.documentIds) { if (input.cancellationSignal?.aborted) break try { - const canonical = await resolveCanonicalActiveKnowledgeDocumentContext({ - knowledgeBaseId: context.knowledgeBaseId, - documentId, - assertedWorkspaceId: context.workspaceId, - }) + const canonical = await resolveCanonicalActiveKnowledgeDocumentContext( + { + knowledgeBaseId: context.knowledgeBaseId, + documentId, + assertedWorkspaceId: context.workspaceId, + }, + principal + ) if (canonical.workspaceId) { await authorizeWorkspaceOperation( principal, @@ -811,7 +880,8 @@ export const bulkDeleteKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ await deleteKnowledgeDocumentInKnowledgeBase( canonical.knowledgeBaseId, canonical.documentId, - generateRequestId() + generateRequestId(), + await context.access.get() ) deletedDocuments.push({ id: canonical.documentId, @@ -860,8 +930,13 @@ export const bulkDeleteKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ export const updateKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.updateDocument, - resolveContext: ({ input }: { input: UpdateKnowledgeDocumentInput }) => - resolveCanonicalActiveKnowledgeDocumentContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: UpdateKnowledgeDocumentInput + }) => resolveCanonicalActiveKnowledgeDocumentContext(input, principal), async execute({ principal, input, context }) { if (input.markFailedDueToTimeout || input.retryProcessing) { const outcome = input.markFailedDueToTimeout @@ -940,14 +1015,20 @@ export const updateKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ export const bulkUpdateKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.bulkDocuments, - resolveContext: ({ input }: { input: BulkKnowledgeDocumentsInput }) => - resolveActiveKnowledgeResourceContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: BulkKnowledgeDocumentsInput + }) => resolveActiveKnowledgeResourceContext(input, principal), async execute({ input, context }) { const result = input.selectAll ? await bulkDocumentOperationByFilter( context.knowledgeBaseId, input.operation, input.enabledFilter, + await context.access.get(), generateRequestId() ) : input.documentIds?.length @@ -955,6 +1036,7 @@ export const bulkUpdateKnowledgeDocuments = defineAuthorizedKnowledgeUseCase({ context.knowledgeBaseId, input.operation, input.documentIds, + await context.access.get(), generateRequestId() ) : null diff --git a/apps/sim/lib/knowledge/application/knowledge-bases.ts b/apps/sim/lib/knowledge/application/knowledge-bases.ts index 7768e23e6d3..95d5f072452 100644 --- a/apps/sim/lib/knowledge/application/knowledge-bases.ts +++ b/apps/sim/lib/knowledge/application/knowledge-bases.ts @@ -57,6 +57,7 @@ import type { KnowledgeOrchestrationResult, } from '@/lib/knowledge/orchestration/shared' import { + attachKnowledgeBaseConnectors, createAuthorizedKnowledgeBase, deleteKnowledgeBase, getKnowledgeBaseById, @@ -345,7 +346,7 @@ async function executeReadKnowledgeBase(args: { { maxRows: MAX_KNOWLEDGE_FOLDERS_PER_WORKSPACE } ) return { - knowledgeBase: args.context.knowledgeBase, + knowledgeBase: await attachKnowledgeBaseConnectors(args.context.knowledgeBase), folderPath: knowledgeFolderPathForId(index, args.context.knowledgeBase.folderId), } } @@ -453,7 +454,13 @@ export const listKnowledgeBaseCatalog = defineAuthorizedKnowledgeUseCase({ */ export const restoreKnowledgeBase = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.restore, - resolveContext: ({ input }: { input: RestoreKnowledgeBaseInput }) => + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: RestoreKnowledgeBaseInput + }) => resolveArchivedKnowledgeBaseContext({ knowledgeBaseId: input.knowledgeBaseId, assertedWorkspaceId: input.assertedWorkspaceId, @@ -561,15 +568,20 @@ export const createKnowledgeBase = defineAuthorizedKnowledgeUseCase({ export const readKnowledgeBase = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.read, - resolveContext: ({ input }: { input: ReadKnowledgeBaseInput }) => - resolveActiveKnowledgeBaseContext(input), + resolveContext: ({ principal, input }: { principal: Principal; input: ReadKnowledgeBaseInput }) => + resolveActiveKnowledgeBaseContext(input, principal), execute: executeReadKnowledgeBase, }) export const updateKnowledgeBaseOperation = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.update, - resolveContext: ({ input }: { input: UpdateKnowledgeBaseInput }) => - resolveActiveKnowledgeBaseContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: UpdateKnowledgeBaseInput + }) => resolveActiveKnowledgeBaseContext(input, principal), execute: executeUpdateKnowledgeBase, projectAudit: ({ input, result }) => ({ action: AuditAction.KNOWLEDGE_BASE_UPDATED, @@ -588,8 +600,13 @@ export const updateKnowledgeBaseOperation = defineAuthorizedKnowledgeUseCase({ export const deleteKnowledgeBaseOperation = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.delete, - resolveContext: ({ input }: { input: DeleteKnowledgeBaseInput }) => - resolveActiveKnowledgeBaseContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: DeleteKnowledgeBaseInput + }) => resolveActiveKnowledgeBaseContext(input, principal), execute: executeDeleteKnowledgeBase, projectAudit: ({ input, result }) => ({ action: AuditAction.KNOWLEDGE_BASE_DELETED, @@ -628,10 +645,13 @@ export const bulkDeleteKnowledgeBases = defineAuthorizedKnowledgeUseCase({ if (input.cancellationSignal?.aborted) break let knowledgeBaseName = knowledgeBaseId try { - const canonical = await resolveActiveKnowledgeBaseContext({ - knowledgeBaseId, - assertedWorkspaceId: context.workspaceId, - }) + const canonical = await resolveActiveKnowledgeBaseContext( + { + knowledgeBaseId, + assertedWorkspaceId: context.workspaceId, + }, + principal + ) knowledgeBaseName = canonical.knowledgeBase.name await authorizeWorkspaceOperation(principal, knowledgeOperations.bulkDelete, canonical, { delegation: knowledgeDelegationPolicy, diff --git a/apps/sim/lib/knowledge/application/knowledge-vfs.ts b/apps/sim/lib/knowledge/application/knowledge-vfs.ts index 8c9dd09e1df..4a7bf4e06fb 100644 --- a/apps/sim/lib/knowledge/application/knowledge-vfs.ts +++ b/apps/sim/lib/knowledge/application/knowledge-vfs.ts @@ -61,7 +61,7 @@ async function resolveKnowledgeBaseByVfsName( context: KnowledgeWorkspaceContext, sourceName: string, sourceSegments?: string[] -): Promise> { +): Promise> { if (sourceSegments && sourceSegments.length > 1) { const row = await resolveResourceRowBySegments( knowledgeVfsAdapter, diff --git a/apps/sim/lib/knowledge/application/operations.test.ts b/apps/sim/lib/knowledge/application/operations.test.ts index 9180ee81346..53cdac8308f 100644 --- a/apps/sim/lib/knowledge/application/operations.test.ts +++ b/apps/sim/lib/knowledge/application/operations.test.ts @@ -55,6 +55,10 @@ describe('knowledge operation registry', () => { 'knowledge.connectors.read', 'knowledge.connectors.create', 'knowledge.connectors.update', + 'knowledge.connectors.access.update', + 'knowledge.connectors.members.list', + 'knowledge.connectors.members.enroll', + 'knowledge.simSearch.connect', 'knowledge.connectors.delete', 'knowledge.connectors.sync', 'knowledge.connectors.documents.list', diff --git a/apps/sim/lib/knowledge/application/operations.ts b/apps/sim/lib/knowledge/application/operations.ts index ddd8cdc6fe8..0228b0d2fd8 100644 --- a/apps/sim/lib/knowledge/application/operations.ts +++ b/apps/sim/lib/knowledge/application/operations.ts @@ -386,6 +386,50 @@ export const knowledgeOperations = { capability: 'knowledge.use', ...HUMAN_AND_COPILOT_PRINCIPAL_POLICY, }), + /** + * Which people a connector crawls as is an admin decision: members mode + * grants the connector every enrolled member's credential. Session only — + * it is a settings action, not something an agent or key performs. + */ + updateConnectorAccess: defineWorkspaceOperation({ + id: 'knowledge.connectors.access.update', + minimumRole: 'admin', + workspaceApiKey: 'deny', + capability: 'knowledge.use', + principalKinds: ['session'], + }), + /** Every per-member connector in the workspace, with where the viewer stands on each. */ + listWorkspaceMemberConnectors: defineWorkspaceOperation({ + id: 'knowledge.connectors.members.list', + minimumRole: 'read', + workspaceApiKey: 'deny', + capability: 'knowledge.use', + principalKinds: ['session'], + }), + /** + * A workspace member joining a per-member connector: any reader may connect + * their own account, which only ever widens what they themselves see. + */ + enrollConnectorMember: defineWorkspaceOperation({ + id: 'knowledge.connectors.members.enroll', + minimumRole: 'read', + workspaceApiKey: 'deny', + capability: 'knowledge.use', + principalKinds: ['session'], + }), + /** + * Connecting a Sim Search source: any reader may connect their own account. + * The first connect of a source also creates its knowledge base and + * connector, which the use case reserves for an admin and refuses to anyone + * else with the way forward (ask an admin to connect the source first). + */ + simSearchConnect: defineWorkspaceOperation({ + id: 'knowledge.simSearch.connect', + minimumRole: 'read', + workspaceApiKey: 'deny', + capability: 'knowledge.use', + principalKinds: ['session'], + }), deleteConnector: defineWorkspaceOperation({ id: 'knowledge.connectors.delete', minimumRole: 'write', diff --git a/apps/sim/lib/knowledge/application/search.test.ts b/apps/sim/lib/knowledge/application/search.test.ts index bd8990759c1..e5566f681bc 100644 --- a/apps/sim/lib/knowledge/application/search.test.ts +++ b/apps/sim/lib/knowledge/application/search.test.ts @@ -40,6 +40,11 @@ vi.mock('@/lib/billing/core/billing-attribution', () => ({ checkAttributedUsageLimits: mocks.checkUsage, })) +/** Retrieval defaults are the flag's concern; here the flag is off so the search stays as configured. */ +vi.mock('@/lib/knowledge/access/availability', () => ({ + isKnowledgeMemberAccessAvailable: async () => false, +})) + vi.mock('@/lib/billing/calculations/usage-monitor', () => ({ checkActorUsageLimits: mocks.checkActorUsage, })) @@ -163,6 +168,7 @@ describe('knowledge search application use case', () => { knowledgeBaseIds: ['knowledge-1'], topK: 5, searchMode: 'vector', + boostRecency: false, }) ) expect(result.results[0]).toMatchObject({ @@ -312,6 +318,43 @@ describe('knowledge search application use case', () => { }) }) + /** + * The provenance snapshot vouches for the name, URL, and tags; the source + * card's modified time and connector type only come from the access-filtered + * metadata read, which a provenance-bearing search must therefore still make. + */ + it('keeps the source card metadata when a provenance registry is present', async () => { + const registry = { markIncomplete: vi.fn() } + const sourceModifiedAt = new Date('2026-08-20T12:00:00Z') + mocks.getDocumentMetadata.mockResolvedValueOnce({ + 'document-1': { + filename: 'guide.pdf', + sourceUrl: 'https://example.com/guide', + sourceModifiedAt, + connectorType: 'google_drive', + }, + }) + + const result = await searchKnowledge.execute({ + principal: { kind: 'session', userId: 'user-1', sessionId: 'session-1' }, + input: { + workspaceId: 'workspace-1', + knowledgeBaseIds: ['knowledge-1'], + query: 'answer', + topK: 5, + resultSecretRegistry: registry as never, + }, + }) + + expect(mocks.getDocumentMetadata).toHaveBeenCalledWith(['document-1'], expect.anything()) + expect(result.results[0]).toMatchObject({ + documentName: 'guide.pdf', + sourceUrl: 'https://example.com/guide', + sourceModifiedAt, + connectorType: 'google_drive', + }) + }) + describe('reranker outcome reporting', () => { const rerankedSearch = (rerankerEnabled?: boolean, query: string | undefined = 'answer') => searchKnowledge.execute({ diff --git a/apps/sim/lib/knowledge/application/search.ts b/apps/sim/lib/knowledge/application/search.ts index c4d492820ac..ae87287303b 100644 --- a/apps/sim/lib/knowledge/application/search.ts +++ b/apps/sim/lib/knowledge/application/search.ts @@ -1,3 +1,4 @@ +import { type Principal, resolvePrincipalSubjectUserId } from '@sim/auth/principal' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { checkActorUsageLimits } from '@/lib/billing/calculations/usage-monitor' @@ -19,6 +20,8 @@ import { isDurableSecretProvenanceEnforced, reportUnrecordedDurableProvenance, } from '@/lib/execution/durable-secret-provenance-enforcement' +import { createKnowledgeAccessProvider } from '@/lib/knowledge/access/scope' +import type { KnowledgeAccessProvider } from '@/lib/knowledge/access/types' import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' import { KnowledgeUsageLimitExceededError, @@ -32,12 +35,13 @@ import { import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { ALL_TAG_SLOTS } from '@/lib/knowledge/constants' import { getEmbeddingModelInfo } from '@/lib/knowledge/embedding-models' +import { generateSearchEmbedding } from '@/lib/knowledge/embeddings' import { runWithKnowledgeModelInputProvenance } from '@/lib/knowledge/model-input-provenance' import { rerank } from '@/lib/knowledge/reranker' import type { RerankerStatus } from '@/lib/knowledge/reranker-models' +import { resolveKnowledgeSearchDefaults } from '@/lib/knowledge/search/defaults' import { executeKnowledgeSearch, - generateSearchEmbedding, getDocumentMetadataByIds, type SearchResult, } from '@/lib/knowledge/search/queries' @@ -101,6 +105,8 @@ export interface SearchKnowledgeInput { type KnowledgeSearchContext = KnowledgeResourceContext & { knowledgeBases: KnowledgeBaseWithCounts[] + /** What the caller may read across the searched bases; resolved from the principal, never from input. */ + access: KnowledgeAccessProvider } export interface KnowledgeSearchItem { @@ -111,6 +117,10 @@ export interface KnowledgeSearchItem { documentId: string documentName: string | null sourceUrl: string | null + /** When the source last changed the document; null for uploads and sources that do not say. */ + sourceModifiedAt: Date | null + /** The connector the document was synced through; null for an upload. */ + connectorType: string | null content: string chunkIndex: number metadata: Record @@ -141,11 +151,14 @@ export interface SearchKnowledgeResult { cost?: KnowledgeSearchCost workspaceId?: string userId: string + /** Whether results were filtered as a person or as the workspace; telemetry only, never presented. */ + accessScopeKind: 'user' | 'workspace' resultSecretRegistry?: ResolvedSecretTraceRegistry } async function resolveKnowledgeSearchContext( - input: SearchKnowledgeInput + input: SearchKnowledgeInput, + principal: Principal ): Promise { if ( input.knowledgeBaseIds.length < 1 || @@ -202,6 +215,7 @@ async function resolveKnowledgeSearchContext( workspaceId: undefined, legacyPersonalOwnerUserId, knowledgeBases: knowledgeBases as KnowledgeBaseWithCounts[], + access: createKnowledgeAccessProvider(principal, {}), } } const workspaceContext = await resolveKnowledgeWorkspaceContext({ @@ -210,13 +224,14 @@ async function resolveKnowledgeSearchContext( return { ...workspaceContext, knowledgeBases: knowledgeBases as KnowledgeBaseWithCounts[], + access: createKnowledgeAccessProvider(principal, { workspaceId: canonicalWorkspaceId }), } } export const searchKnowledge = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.search, - resolveContext: ({ input }: { input: SearchKnowledgeInput }) => - resolveKnowledgeSearchContext(input), + resolveContext: ({ principal, input }: { principal: Principal; input: SearchKnowledgeInput }) => + resolveKnowledgeSearchContext(input, principal), async execute({ principal, input, context }) { const requestId = generateRequestId() const hasQuery = Boolean(input.query?.trim()) @@ -276,6 +291,14 @@ export const searchKnowledge = defineAuthorizedKnowledgeUseCase({ generateSearchEmbedding(input.query!, embeddingModel, context.workspaceId) ) : Promise.resolve(null) + /** Resolved alongside the embedding call; both are needed before the first leg runs. */ + const accessPromise = context.access.get() + const searchDefaults = await resolveKnowledgeSearchDefaults({ + workspaceId: context.workspaceId, + /** The signed-in person, if any; never the billing owner or a key's creator. */ + userId: resolvePrincipalSubjectUserId(principal) ?? undefined, + requestedMode: input.searchMode, + }) const useReranker = Boolean(input.rerankerEnabled && hasQuery) const candidateTopK = useReranker ? input.rerankerInputCount !== undefined @@ -285,10 +308,13 @@ export const searchKnowledge = defineAuthorizedKnowledgeUseCase({ ) : Math.min(KNOWLEDGE_SEARCH_COST_POLICY.maxTopK, input.topK * 4) : input.topK + const access = await accessPromise let rows = await executeKnowledgeSearch({ knowledgeBaseIds, topK: candidateTopK, - searchMode: input.searchMode ?? 'vector', + access, + searchMode: searchDefaults.searchMode, + boostRecency: searchDefaults.boostRecency, query: input.query, queryVector: hasQuery ? JSON.stringify((await queryEmbeddingPromise)?.embedding ?? null) @@ -460,14 +486,21 @@ export const searchKnowledge = defineAuthorizedKnowledgeUseCase({ }) ) const tagMaps = new Map(tagDefinitionEntries) - const basicDocumentMetadata = provenanceSnapshot - ? {} - : await getDocumentMetadataByIds(rows.map((row) => row.documentId)) + /** + * Always read: the provenance snapshot vouches for the name, URL, and tags + * a model may see, but the source card's modified time and connector type + * are only carried here, under the same access predicate as the search. + */ + const basicDocumentMetadata = await getDocumentMetadataByIds( + rows.map((row) => row.documentId), + access + ) const results = rows.map((row): KnowledgeSearchItem => { const metadata: Record = {} const tagMap = tagMaps.get(row.knowledgeBaseId) const provenanceDocument = provenanceSnapshot?.documentMetadata[row.documentId] - const document = provenanceDocument ?? basicDocumentMetadata[row.documentId] + const basicDocument = basicDocumentMetadata[row.documentId] + const document = provenanceDocument ?? basicDocument for (const slot of ALL_TAG_SLOTS) { const value = provenanceDocument && slot.startsWith('tag') @@ -484,6 +517,8 @@ export const searchKnowledge = defineAuthorizedKnowledgeUseCase({ documentId: row.documentId, documentName: document?.filename ?? null, sourceUrl: document?.sourceUrl ?? null, + sourceModifiedAt: basicDocument?.sourceModifiedAt ?? null, + connectorType: basicDocument?.connectorType ?? null, content: row.content, chunkIndex: row.chunkIndex, metadata, @@ -563,6 +598,7 @@ export const searchKnowledge = defineAuthorizedKnowledgeUseCase({ cost, ...(context.workspaceId ? { workspaceId: context.workspaceId } : {}), userId, + accessScopeKind: access.kind, resultSecretRegistry: registry, } }, @@ -571,6 +607,7 @@ export const searchKnowledge = defineAuthorizedKnowledgeUseCase({ knowledgeBaseId: result.knowledgeBaseId, resultsCount: result.totalResults, workspaceId: context.workspaceId, + accessScopeKind: result.accessScopeKind, }) }, }) diff --git a/apps/sim/lib/knowledge/application/sim-search.test.ts b/apps/sim/lib/knowledge/application/sim-search.test.ts new file mode 100644 index 00000000000..8ec011f4310 --- /dev/null +++ b/apps/sim/lib/knowledge/application/sim-search.test.ts @@ -0,0 +1,292 @@ +/** + * @vitest-environment node + */ + +import { knowledgeBase, knowledgeConnector } from '@sim/db/schema' +import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' +import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + resolveWorkspace: vi.fn(), + resolvePermission: vi.fn(), + isMemberAccessAvailable: vi.fn(), + createKnowledgeBase: vi.fn(), + deleteKnowledgeBase: vi.fn(), + createConnector: vi.fn(), + deleteConnector: vi.fn(), + enroll: vi.fn(), + getUserPermissionConfig: vi.fn(), + recordAudit: vi.fn(), +})) + +vi.mock('@sim/audit', () => ({ + AuditAction: {}, + AuditResourceType: {}, + recordAudit: mocks.recordAudit, +})) + +vi.mock('@sim/platform-authz/workspace', () => ({ + permissionSatisfies: (actual: string | null, required: string) => { + const rank = { read: 1, write: 2, admin: 3 } as const + return ( + actual !== null && rank[actual as keyof typeof rank] >= rank[required as keyof typeof rank] + ) + }, + resolveEffectiveWorkspacePermission: mocks.resolvePermission, +})) + +vi.mock('@/lib/knowledge/application/contexts', () => ({ + resolveKnowledgeWorkspaceContext: mocks.resolveWorkspace, +})) + +vi.mock('@/lib/knowledge/access/availability', async () => { + const { OrchestrationError } = await import('@/lib/core/orchestration/types') + return { + isKnowledgeMemberAccessAvailable: mocks.isMemberAccessAvailable, + requireKnowledgeMemberAccessAvailable: async (context: { workspaceId: string }) => { + if (await mocks.isMemberAccessAvailable(context)) return + throw new OrchestrationError( + 'validation', + 'Per-member access is not available for this workspace' + ) + }, + } +}) + +vi.mock('@/lib/knowledge/application/knowledge-bases', () => ({ + createKnowledgeBase: { execute: mocks.createKnowledgeBase }, + deleteKnowledgeBaseOperation: { execute: mocks.deleteKnowledgeBase }, +})) + +vi.mock('@/lib/knowledge/application/connectors', () => ({ + createKnowledgeConnector: { execute: mocks.createConnector }, + deleteKnowledgeConnector: { execute: mocks.deleteConnector }, +})) + +vi.mock('@/lib/knowledge/application/connector-access', () => ({ + startKnowledgeConnectorMemberEnrollment: { execute: mocks.enroll }, +})) + +vi.mock('@/lib/permission-groups/resolve.server', () => ({ + getUserPermissionConfig: mocks.getUserPermissionConfig, +})) + +vi.mock('@/lib/sim-search/connectors', () => ({ + SIM_SEARCH_KNOWLEDGE_BASE_NAME: 'Sim Search', + canConnectPersonally: (meta: { permissionScopedListing?: unknown }) => + Boolean(meta.permissionScopedListing), + missingSetupFields: ( + meta: { configFields: Array<{ id: string; title: string; required?: boolean }> }, + sourceConfig: Record + ) => + meta.configFields.filter( + (field) => field.required && typeof sourceConfig[field.id] !== 'string' + ), +})) + +vi.mock('@/connectors/registry', () => ({ + CONNECTOR_META_REGISTRY: { + google_drive: { + name: 'Google Drive', + auth: { mode: 'oauth', provider: 'google-drive' }, + permissionScopedListing: { capFieldIds: [] }, + configFields: [], + }, + confluence: { + name: 'Confluence', + auth: { mode: 'oauth', provider: 'confluence' }, + permissionScopedListing: { capFieldIds: [] }, + configFields: [{ id: 'spaceKey', title: 'a space key', required: true }], + }, + }, +})) + +import { connectSimSearchConnector } from '@/lib/knowledge/application/sim-search' +import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/fields' + +const workspaceContext = { + workspaceId: 'workspace-1', + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + billedAccountUserId: 'owner-1', +} + +const principal = { kind: 'session' as const, userId: 'user-1', sessionId: 'session-1' } +const existingConnector = { knowledgeBaseId: 'kb-search', connectorId: 'connector-drive' } + +/** The first lookup runs before the coalesced creation and the second inside it. */ +function queueConnectorLookups(...results: Array) { + for (const result of results) { + queueTableRows(knowledgeConnector, result ? [result] : []) + } +} + +describe('connectSimSearchConnector', () => { + afterAll(resetDbChainMock) + + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mocks.resolveWorkspace.mockResolvedValue(workspaceContext) + mocks.getUserPermissionConfig.mockResolvedValue(DEFAULT_PERMISSION_GROUP_CONFIG) + mocks.isMemberAccessAvailable.mockResolvedValue(true) + mocks.createKnowledgeBase.mockResolvedValue({ knowledgeBase: { id: 'kb-new' } }) + mocks.createConnector.mockResolvedValue({ connector: { id: 'connector-new' } }) + mocks.enroll.mockResolvedValue({ url: 'https://sim.test/enroll/token' }) + }) + + it('enrolls a reader in a source someone already connected', async () => { + mocks.resolvePermission.mockResolvedValue('read') + queueConnectorLookups(existingConnector) + + const result = await connectSimSearchConnector.execute({ + principal, + input: { workspaceId: 'workspace-1', connectorType: 'google_drive' }, + }) + + expect(result).toEqual({ ...existingConnector, url: 'https://sim.test/enroll/token' }) + expect(mocks.createKnowledgeBase).not.toHaveBeenCalled() + expect(mocks.createConnector).not.toHaveBeenCalled() + expect(mocks.enroll).toHaveBeenCalledWith( + expect.objectContaining({ + principal, + input: { ...existingConnector, assertedWorkspaceId: 'workspace-1' }, + }) + ) + }) + + it('tells a reader to ask an admin when the source has no connector yet', async () => { + mocks.resolvePermission.mockResolvedValue('read') + queueConnectorLookups(null) + + await expect( + connectSimSearchConnector.execute({ + principal, + input: { workspaceId: 'workspace-1', connectorType: 'google_drive' }, + }) + ).rejects.toMatchObject({ + code: 'forbidden', + message: expect.stringContaining('Ask a workspace admin to connect Google Drive first'), + }) + expect(mocks.createKnowledgeBase).not.toHaveBeenCalled() + expect(mocks.createConnector).not.toHaveBeenCalled() + expect(mocks.enroll).not.toHaveBeenCalled() + }) + + it('refuses before creating anything when per-member access is unavailable', async () => { + mocks.resolvePermission.mockResolvedValue('admin') + mocks.isMemberAccessAvailable.mockResolvedValue(false) + queueConnectorLookups(null) + + await expect( + connectSimSearchConnector.execute({ + principal, + input: { workspaceId: 'workspace-1', connectorType: 'google_drive' }, + }) + ).rejects.toMatchObject({ code: 'validation' }) + expect(mocks.createKnowledgeBase).not.toHaveBeenCalled() + expect(mocks.createConnector).not.toHaveBeenCalled() + }) + + it('lets an admin create the base and the connector with the setup fields, then enrolls them', async () => { + mocks.resolvePermission.mockResolvedValue('admin') + queueTableRows(knowledgeBase, []) + queueTableRows(knowledgeBase, [{ id: 'kb-new' }]) + queueConnectorLookups(null, null, { + knowledgeBaseId: 'kb-new', + connectorId: 'connector-new', + } as typeof existingConnector) + + const result = await connectSimSearchConnector.execute({ + principal, + input: { + workspaceId: 'workspace-1', + connectorType: 'confluence', + sourceConfig: { spaceKey: 'ENG' }, + }, + }) + + expect(mocks.deleteKnowledgeBase).not.toHaveBeenCalled() + expect(mocks.deleteConnector).not.toHaveBeenCalled() + + expect(mocks.createKnowledgeBase).toHaveBeenCalledWith( + expect.objectContaining({ + principal, + input: expect.objectContaining({ workspaceId: 'workspace-1', name: 'Sim Search' }), + }) + ) + expect(mocks.createConnector).toHaveBeenCalledWith( + expect.objectContaining({ + principal, + input: expect.objectContaining({ + knowledgeBaseId: 'kb-new', + assertedWorkspaceId: 'workspace-1', + connectorType: 'confluence', + sourceConfig: { spaceKey: 'ENG' }, + accessMode: 'members', + }), + }) + ) + expect(dbChainMockFns.transaction).not.toHaveBeenCalled() + expect(result).toEqual({ + knowledgeBaseId: 'kb-new', + connectorId: 'connector-new', + url: 'https://sim.test/enroll/token', + }) + }) + + it('refuses a first connect that leaves a setup field empty', async () => { + mocks.resolvePermission.mockResolvedValue('admin') + queueConnectorLookups(null) + + await expect( + connectSimSearchConnector.execute({ + principal, + input: { workspaceId: 'workspace-1', connectorType: 'confluence' }, + }) + ).rejects.toMatchObject({ + code: 'validation', + message: 'Confluence needs a space key to connect', + }) + expect(mocks.createKnowledgeBase).not.toHaveBeenCalled() + }) + + it('reuses the connector another first connect created while it waited', async () => { + mocks.resolvePermission.mockResolvedValue('admin') + queueConnectorLookups(null, existingConnector) + queueTableRows(knowledgeBase, [{ id: existingConnector.knowledgeBaseId }]) + + const result = await connectSimSearchConnector.execute({ + principal, + input: { workspaceId: 'workspace-1', connectorType: 'google_drive' }, + }) + + expect(mocks.createKnowledgeBase).not.toHaveBeenCalled() + expect(mocks.createConnector).not.toHaveBeenCalled() + expect(result).toEqual({ ...existingConnector, url: 'https://sim.test/enroll/token' }) + }) + + it('converges on the row another instance created first and deletes its own', async () => { + mocks.resolvePermission.mockResolvedValue('admin') + queueTableRows(knowledgeBase, []) + queueTableRows(knowledgeBase, [{ id: existingConnector.knowledgeBaseId }]) + queueConnectorLookups(null, null, existingConnector) + + const result = await connectSimSearchConnector.execute({ + principal, + input: { workspaceId: 'workspace-1', connectorType: 'google_drive' }, + }) + + expect(mocks.deleteKnowledgeBase).toHaveBeenCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ knowledgeBaseId: 'kb-new' }), + }) + ) + expect(mocks.deleteConnector).toHaveBeenCalledWith( + expect.objectContaining({ + input: expect.objectContaining({ connectorId: 'connector-new', deleteDocuments: true }), + }) + ) + expect(result).toEqual({ ...existingConnector, url: 'https://sim.test/enroll/token' }) + }) +}) diff --git a/apps/sim/lib/knowledge/application/sim-search.ts b/apps/sim/lib/knowledge/application/sim-search.ts new file mode 100644 index 00000000000..5123723e4d1 --- /dev/null +++ b/apps/sim/lib/knowledge/application/sim-search.ts @@ -0,0 +1,240 @@ +import { resolvePrincipalSubjectUserId } from '@sim/auth/principal' +import { db } from '@sim/db' +import { knowledgeBase, knowledgeConnector } from '@sim/db/schema' +import { and, asc, eq, isNull } from 'drizzle-orm' +import { coalesceLocally } from '@/lib/concurrency/singleflight' +import { + InsufficientWorkspacePermissionsError, + requireCurrentHumanRole, +} from '@/lib/core/application/workspace-authorization' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { requireKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' +import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' +import { startKnowledgeConnectorMemberEnrollment } from '@/lib/knowledge/application/connector-access' +import { + createKnowledgeConnector, + deleteKnowledgeConnector, +} from '@/lib/knowledge/application/connectors' +import { + type KnowledgeWorkspaceContext, + resolveKnowledgeWorkspaceContext, +} from '@/lib/knowledge/application/contexts' +import { + createKnowledgeBase, + deleteKnowledgeBaseOperation, +} from '@/lib/knowledge/application/knowledge-bases' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { + canConnectPersonally, + missingSetupFields, + SIM_SEARCH_KNOWLEDGE_BASE_NAME, +} from '@/lib/sim-search/connectors' +import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' + +const SIM_SEARCH_KNOWLEDGE_BASE_DESCRIPTION = + 'What each person can open in the sources they connected, searched as them.' +/** Between runs the change feeds keep deletions and unshares fresh; the hourly run fills the rest. */ +const SIM_SEARCH_SYNC_INTERVAL_MINUTES = 60 + +export interface ConnectSimSearchConnectorInput { + workspaceId: string + /** `CONNECTOR_META_REGISTRY` key of the source to connect. */ + connectorType: string + /** The source's setup fields (a site, a space); read only when this connect creates the connector. */ + sourceConfig?: Record +} + +export interface ConnectSimSearchConnectorResult { + knowledgeBaseId: string + connectorId: string + /** The enrollment link that connects the caller's own account. */ + url: string +} + +async function findSimSearchConnector(workspaceId: string, connectorType: string) { + const [row] = await db + .select({ knowledgeBaseId: knowledgeBase.id, connectorId: knowledgeConnector.id }) + .from(knowledgeConnector) + .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) + .where( + and( + eq(knowledgeBase.workspaceId, workspaceId), + eq(knowledgeBase.name, SIM_SEARCH_KNOWLEDGE_BASE_NAME), + isNull(knowledgeBase.deletedAt), + eq(knowledgeConnector.connectorType, connectorType), + eq(knowledgeConnector.accessMode, 'members'), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + .orderBy(asc(knowledgeConnector.createdAt)) + .limit(1) + return row ?? null +} + +async function findSimSearchKnowledgeBase(workspaceId: string) { + const [row] = await db + .select({ id: knowledgeBase.id }) + .from(knowledgeBase) + .where( + and( + eq(knowledgeBase.workspaceId, workspaceId), + eq(knowledgeBase.name, SIM_SEARCH_KNOWLEDGE_BASE_NAME), + isNull(knowledgeBase.deletedAt) + ) + ) + .orderBy(asc(knowledgeBase.createdAt)) + .limit(1) + return row ?? null +} + +/** + * The first connect of a source turns it on for the whole workspace, which is + * an admin decision the same way a members-mode connector is. Refused with + * the way forward rather than the nested operations' generic role error, so a + * reader learns whom to ask and for what. + */ +async function requireSimSearchSetupAdmin( + userId: string, + context: KnowledgeWorkspaceContext, + sourceName: string +): Promise { + try { + await requireCurrentHumanRole(userId, context, 'admin') + } catch (error) { + if (!(error instanceof InsufficientWorkspacePermissionsError)) throw error + throw new OrchestrationError( + 'forbidden', + `${sourceName} is not connected in this workspace yet. Ask a workspace admin to connect ${sourceName} first; after that everyone connects their own account.` + ) + } +} + +/** + * One click on a Sim Search source: the workspace's Sim Search knowledge base + * and a per-member connector for that source exist after this, and the caller + * gets the link that connects their own account. The first connect of a + * source creates both, which takes a workspace admin and the source's setup + * fields when it has any; every connect after that only enrolls. The OAuth + * completion queues the member run, so indexing starts on its own. + * + * The creating branch shares one creation per workspace base and per source + * within the process and re-checks before creating: nothing in the schema + * keeps two concurrent first connects from each creating a Sim Search base + * and a connector of the same source. The nested use cases query the pool, + * so this cannot hold a transaction across them. + */ +export const connectSimSearchConnector = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.simSearchConnect, + resolveContext: ({ input }: { input: ConnectSimSearchConnectorInput }) => + resolveKnowledgeWorkspaceContext(input), + async execute({ principal, input, context, request }): Promise { + const meta = CONNECTOR_META_REGISTRY[input.connectorType] + if (!meta || !canConnectPersonally(meta)) { + throw new OrchestrationError( + 'validation', + 'This source cannot be connected per person; a workspace admin sets it up from a knowledge base' + ) + } + const workspaceId = context.workspaceId + let target = await findSimSearchConnector(workspaceId, input.connectorType) + if (!target) { + const userId = resolvePrincipalSubjectUserId(principal) + if (!userId) throw new OrchestrationError('forbidden', 'Sign in to connect your account') + const sourceConfig = input.sourceConfig ?? {} + const missing = missingSetupFields(meta, sourceConfig) + if (missing.length > 0) { + throw new OrchestrationError( + 'validation', + `${meta.name} needs ${missing.map((field) => field.title).join(' and ')} to connect` + ) + } + /** + * Judged before anything is created: the connector creation below checks + * the same availability, but only after the knowledge base exists. + */ + await Promise.all([ + requireKnowledgeMemberAccessAvailable({ workspaceId }), + requireSimSearchSetupAdmin(userId, context, meta.name), + ]) + /** + * Concurrent first connects in this process share one creation per + * workspace base and per source, and each re-checks before creating. + * Two instances can still race in the same instant: every lookup here + * orders by the oldest row, so after creating, each re-reads and the one + * that finds an older row than its own deletes what it just made and + * converges on the older one. Nothing stray outlives the request. + */ + const knowledgeBaseId = await coalesceLocally(`sim-search:base:${workspaceId}`, async () => { + const existing = await findSimSearchKnowledgeBase(workspaceId) + if (existing) return existing.id + const created = ( + await createKnowledgeBase.execute({ + principal, + input: { + workspaceId, + name: SIM_SEARCH_KNOWLEDGE_BASE_NAME, + description: SIM_SEARCH_KNOWLEDGE_BASE_DESCRIPTION, + source: 'ui', + }, + request, + }) + ).knowledgeBase.id + const oldest = (await findSimSearchKnowledgeBase(workspaceId))?.id ?? created + if (oldest !== created) { + await deleteKnowledgeBaseOperation.execute({ + principal, + input: { knowledgeBaseId: created, assertedWorkspaceId: workspaceId, source: 'ui' }, + request, + }) + } + return oldest + }) + target = await coalesceLocally( + `sim-search:connect:${workspaceId}:${input.connectorType}`, + async () => { + const existing = await findSimSearchConnector(workspaceId, input.connectorType) + if (existing) return existing + const created = await createKnowledgeConnector.execute({ + principal, + input: { + knowledgeBaseId, + assertedWorkspaceId: workspaceId, + connectorType: input.connectorType, + sourceConfig, + syncIntervalMinutes: SIM_SEARCH_SYNC_INTERVAL_MINUTES, + accessMode: 'members', + source: 'ui', + }, + request, + }) + const oldest = await findSimSearchConnector(workspaceId, input.connectorType) + if (oldest && oldest.connectorId !== created.connector.id) { + await deleteKnowledgeConnector.execute({ + principal, + input: { + connectorId: created.connector.id, + assertedWorkspaceId: workspaceId, + deleteDocuments: true, + source: 'ui', + }, + request, + }) + return oldest + } + return { knowledgeBaseId, connectorId: created.connector.id } + } + ) + } + const { url } = await startKnowledgeConnectorMemberEnrollment.execute({ + principal, + input: { + knowledgeBaseId: target.knowledgeBaseId, + connectorId: target.connectorId, + assertedWorkspaceId: workspaceId, + }, + request, + }) + return { ...target, url } + }, +}) diff --git a/apps/sim/lib/knowledge/application/tags.test.ts b/apps/sim/lib/knowledge/application/tags.test.ts index a0695189cff..19101ed687f 100644 --- a/apps/sim/lib/knowledge/application/tags.test.ts +++ b/apps/sim/lib/knowledge/application/tags.test.ts @@ -59,6 +59,7 @@ vi.mock('@/lib/knowledge/tags/service', () => ({ deleteAllTagDefinitions: mocks.deleteAllTags, })) +import { WORKSPACE_ACCESS_SCOPE } from '@/lib/knowledge/access/scope' import { createKnowledgeTag, deleteKnowledgeDocumentTagDefinitions, @@ -70,7 +71,11 @@ import { updateKnowledgeTag, } from '@/lib/knowledge/application/tags' +/** Every mocked context carries the workspace read scope the resolvers would attach. */ +const knowledgeAccess = { get: async () => WORKSPACE_ACCESS_SCOPE } + const crossWorkspaceContext = { + access: knowledgeAccess, workspaceId: 'workspace-b', workspaceOrganizationId: null, allowPersonalApiKeys: true, @@ -589,7 +594,8 @@ describe('knowledge tag application use cases', () => { }) expect(mocks.resolveKnowledgeBase).toHaveBeenCalledWith( - expect.objectContaining({ knowledgeBaseId: 'knowledge-b' }) + expect.objectContaining({ knowledgeBaseId: 'knowledge-b' }), + expect.objectContaining({ kind: 'session' }) ) expect(mocks.resolveDocument).not.toHaveBeenCalled() expect(mocks.recordAudit).toHaveBeenCalledWith( diff --git a/apps/sim/lib/knowledge/application/tags.ts b/apps/sim/lib/knowledge/application/tags.ts index 8c3251f6dc5..c2e836f1075 100644 --- a/apps/sim/lib/knowledge/application/tags.ts +++ b/apps/sim/lib/knowledge/application/tags.ts @@ -1,4 +1,5 @@ import { AuditAction, AuditResourceType } from '@sim/audit' +import type { Principal } from '@sim/auth/principal' import { getPostgresConstraintName, getPostgresErrorCode } from '@sim/utils/errors' import { OrchestrationError } from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' @@ -133,8 +134,8 @@ function tagUniquenessConflict(error: unknown): never { export const listKnowledgeTags = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.listTags, - resolveContext: ({ input }: { input: ListKnowledgeTagsInput }) => - resolveActiveKnowledgeResourceContext(input), + resolveContext: ({ principal, input }: { principal: Principal; input: ListKnowledgeTagsInput }) => + resolveActiveKnowledgeResourceContext(input, principal), async execute({ context }) { return { tagDefinitions: await getDocumentTagDefinitions(context.knowledgeBaseId) } }, @@ -142,8 +143,13 @@ export const listKnowledgeTags = defineAuthorizedKnowledgeUseCase({ export const createKnowledgeTag = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.createTag, - resolveContext: ({ input }: { input: CreateKnowledgeTagInput }) => - resolveActiveKnowledgeResourceContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: CreateKnowledgeTagInput + }) => resolveActiveKnowledgeResourceContext(input, principal), async execute({ input, context }): Promise<{ tagDefinition: TagDefinition knowledgeBaseId: string @@ -222,8 +228,13 @@ export const createKnowledgeTag = defineAuthorizedKnowledgeUseCase({ export const updateKnowledgeTag = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.updateTag, - resolveContext: ({ input }: { input: UpdateKnowledgeTagInput }) => - resolveActiveKnowledgeTagContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: UpdateKnowledgeTagInput + }) => resolveActiveKnowledgeTagContext(input, principal), async execute({ input, context }): Promise<{ tagDefinition: TagDefinition knowledgeBaseId: string @@ -308,8 +319,13 @@ export const updateKnowledgeTag = defineAuthorizedKnowledgeUseCase({ export const deleteKnowledgeTag = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.deleteTag, - resolveContext: ({ input }: { input: DeleteKnowledgeTagInput }) => - resolveActiveKnowledgeTagContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: DeleteKnowledgeTagInput + }) => resolveActiveKnowledgeTagContext(input, principal), async execute({ context }) { const deleted = await deleteTagDefinition( context.knowledgeBaseId, @@ -335,26 +351,43 @@ export const deleteKnowledgeTag = defineAuthorizedKnowledgeUseCase({ export const readKnowledgeTagUsage = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.readTagUsage, - resolveContext: ({ input }: { input: ListKnowledgeTagsInput }) => - resolveActiveKnowledgeResourceContext(input), + resolveContext: ({ principal, input }: { principal: Principal; input: ListKnowledgeTagsInput }) => + resolveActiveKnowledgeResourceContext(input, principal), async execute({ context }) { - return { usage: await getTagUsageStats(context.knowledgeBaseId, generateRequestId()) } + return { + usage: await getTagUsageStats( + context.knowledgeBaseId, + await context.access.get(), + generateRequestId() + ), + } }, }) export const readDetailedKnowledgeTagUsage = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.readDetailedTagUsage, - resolveContext: ({ input }: { input: ListKnowledgeTagsInput }) => - resolveActiveKnowledgeResourceContext(input), + resolveContext: ({ principal, input }: { principal: Principal; input: ListKnowledgeTagsInput }) => + resolveActiveKnowledgeResourceContext(input, principal), async execute({ context }) { - return { usage: await getTagUsage(context.knowledgeBaseId, generateRequestId()) } + return { + usage: await getTagUsage( + context.knowledgeBaseId, + generateRequestId(), + await context.access.get() + ), + } }, }) export const readNextKnowledgeTagSlot = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.readNextTagSlot, - resolveContext: ({ input }: { input: ReadNextKnowledgeTagSlotInput }) => - resolveActiveKnowledgeResourceContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: ReadNextKnowledgeTagSlotInput + }) => resolveActiveKnowledgeResourceContext(input, principal), async execute({ input, context }) { const fieldType = SUPPORTED_FIELD_TYPES.find((supported) => supported === input.fieldType) if (!fieldType) { @@ -391,8 +424,13 @@ export const readNextKnowledgeTagSlot = defineAuthorizedKnowledgeUseCase({ export const listKnowledgeDocumentTagDefinitions = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.listTags, - resolveContext: ({ input }: { input: KnowledgeDocumentTagDefinitionsInput }) => - resolveCanonicalActiveKnowledgeDocumentContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: KnowledgeDocumentTagDefinitionsInput + }) => resolveCanonicalActiveKnowledgeDocumentContext(input, principal), async execute({ context }) { return { tagDefinitions: await getDocumentTagDefinitions(context.knowledgeBaseId) } }, @@ -408,8 +446,13 @@ export const listKnowledgeDocumentTagDefinitions = defineAuthorizedKnowledgeUseC */ export const saveKnowledgeDocumentTagDefinitions = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.saveDocumentTagDefinitions, - resolveContext: ({ input }: { input: SaveKnowledgeDocumentTagDefinitionsInput }) => - resolveActiveKnowledgeResourceContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: SaveKnowledgeDocumentTagDefinitionsInput + }) => resolveActiveKnowledgeResourceContext(input, principal), async execute({ input, context }) { for (const definition of input.definitions) { if (!(SUPPORTED_FIELD_TYPES as readonly string[]).includes(definition.fieldType)) { @@ -450,8 +493,13 @@ export const saveKnowledgeDocumentTagDefinitions = defineAuthorizedKnowledgeUseC */ export const deleteKnowledgeDocumentTagDefinitions = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.deleteDocumentTagDefinitions, - resolveContext: ({ input }: { input: DeleteKnowledgeDocumentTagDefinitionsInput }) => - resolveActiveKnowledgeResourceContext(input), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: DeleteKnowledgeDocumentTagDefinitionsInput + }) => resolveActiveKnowledgeResourceContext(input, principal), async execute({ input, context }) { if (input.action === 'cleanup') { return { diff --git a/apps/sim/lib/knowledge/application/upload-sessions.ts b/apps/sim/lib/knowledge/application/upload-sessions.ts index 7abafbfe954..67789d30112 100644 --- a/apps/sim/lib/knowledge/application/upload-sessions.ts +++ b/apps/sim/lib/knowledge/application/upload-sessions.ts @@ -88,11 +88,20 @@ export interface CompleteKnowledgeDocumentUploadResult { export const createKnowledgeDocumentUpload = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.uploadCreate, - resolveContext: ({ input }: { input: CreateKnowledgeDocumentUploadInput }) => - resolveActiveKnowledgeBaseContext({ - knowledgeBaseId: input.knowledgeBaseId, - assertedWorkspaceId: input.assertedWorkspaceId, - }), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: CreateKnowledgeDocumentUploadInput + }) => + resolveActiveKnowledgeBaseContext( + { + knowledgeBaseId: input.knowledgeBaseId, + assertedWorkspaceId: input.assertedWorkspaceId, + }, + principal + ), async execute({ principal, input, context, request }) { if (!request) throw new Error('Knowledge upload creation requires a request context') const billingAttribution = await resolveKnowledgeBillingAttribution(principal, context) @@ -140,11 +149,20 @@ export const createKnowledgeDocumentUpload = defineAuthorizedKnowledgeUseCase({ export const issueKnowledgeDocumentUploadParts = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.uploadParts, - resolveContext: ({ input }: { input: IssueKnowledgeDocumentUploadPartsInput }) => - resolveActiveKnowledgeBaseContext({ - knowledgeBaseId: input.knowledgeBaseId, - assertedWorkspaceId: input.assertedWorkspaceId, - }), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: IssueKnowledgeDocumentUploadPartsInput + }) => + resolveActiveKnowledgeBaseContext( + { + knowledgeBaseId: input.knowledgeBaseId, + assertedWorkspaceId: input.assertedWorkspaceId, + }, + principal + ), async execute({ principal, input, context, request }) { if (!request) throw new Error('Knowledge upload part issuance requires a request context') const session = await loadBoundKnowledgeDocumentUpload(principal, input, context) @@ -161,11 +179,20 @@ export const issueKnowledgeDocumentUploadParts = defineAuthorizedKnowledgeUseCas export const cancelKnowledgeDocumentUpload = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.uploadCancel, - resolveContext: ({ input }: { input: KnowledgeDocumentUploadControlInput }) => - resolveActiveKnowledgeBaseContext({ - knowledgeBaseId: input.knowledgeBaseId, - assertedWorkspaceId: input.assertedWorkspaceId, - }), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: KnowledgeDocumentUploadControlInput + }) => + resolveActiveKnowledgeBaseContext( + { + knowledgeBaseId: input.knowledgeBaseId, + assertedWorkspaceId: input.assertedWorkspaceId, + }, + principal + ), async execute({ principal, input, context }) { const session = await loadBoundKnowledgeDocumentUpload(principal, input, context) await reauthorizeKnowledgeDocumentUpload(principal, session, knowledgeOperations.uploadCancel) @@ -183,11 +210,20 @@ export const cancelKnowledgeDocumentUpload = defineAuthorizedKnowledgeUseCase({ export const completeKnowledgeDocumentUpload = defineAuthorizedKnowledgeUseCase({ operation: knowledgeOperations.uploadComplete, - resolveContext: ({ input }: { input: CompleteKnowledgeDocumentUploadInput }) => - resolveActiveKnowledgeBaseContext({ - knowledgeBaseId: input.knowledgeBaseId, - assertedWorkspaceId: input.assertedWorkspaceId, - }), + resolveContext: ({ + principal, + input, + }: { + principal: Principal + input: CompleteKnowledgeDocumentUploadInput + }) => + resolveActiveKnowledgeBaseContext( + { + knowledgeBaseId: input.knowledgeBaseId, + assertedWorkspaceId: input.assertedWorkspaceId, + }, + principal + ), async execute({ principal, input, @@ -446,10 +482,13 @@ async function reauthorizeKnowledgeDocumentUpload( throw new OrchestrationError('not_found', 'Upload session not found') } assertUploadSessionAuthBinding(session, principal) - const context = await resolveActiveKnowledgeBaseContext({ - knowledgeBaseId: session.knowledgeBaseId, - assertedWorkspaceId: session.workspaceId, - }) + const context = await resolveActiveKnowledgeBaseContext( + { + knowledgeBaseId: session.knowledgeBaseId, + assertedWorkspaceId: session.workspaceId, + }, + principal + ) await authorizeWorkspaceOperation(principal, operation, context, { delegation: knowledgeDelegationPolicy, }) diff --git a/apps/sim/lib/knowledge/chunks/keyset-sql.test.ts b/apps/sim/lib/knowledge/chunks/keyset-sql.test.ts index d545ed7e60c..2dfacc51ff2 100644 --- a/apps/sim/lib/knowledge/chunks/keyset-sql.test.ts +++ b/apps/sim/lib/knowledge/chunks/keyset-sql.test.ts @@ -9,6 +9,8 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' vi.unmock('@sim/db/schema') vi.unmock('drizzle-orm') +import { WORKSPACE_ACCESS_SCOPE } from '@/lib/knowledge/access/scope' +import { SYSTEM_ACCESS_SCOPE } from '@/lib/knowledge/access/types' import { queryChunks } from '@/lib/knowledge/chunks/service' import type { ChunkSortBy } from '@/lib/knowledge/chunks/types' @@ -39,7 +41,8 @@ async function readPageSql(sortBy: ChunkSortBy, sortOrder: 'asc' | 'desc', curso await queryChunks( 'document-1', { sortBy, sortOrder, limit: 10, cursorKeys: cursorKeys as never }, - 'request-1' + 'request-1', + SYSTEM_ACCESS_SCOPE ) return { where: render(dbChainMockFns.where.mock.calls[0]?.[0]), @@ -90,10 +93,18 @@ describe('chunk list generated SQL', () => { ) it('binds a search term with its LIKE wildcards escaped', async () => { - await queryChunks('document-1', { search: '100%_raw\\' }, 'request-1') + await queryChunks('document-1', { search: '100%_raw\\' }, 'request-1', SYSTEM_ACCESS_SCOPE) const where = render(dbChainMockFns.where.mock.calls[0]?.[0]) expect(where.sql).toContain('"embedding"."content" ilike $') expect(where.params).toContain('%100\\%\\_raw\\\\%') }) + + it('binds the caller access tokens as scalars against the document acl', async () => { + await queryChunks('document-1', {}, 'request-1', WORKSPACE_ACCESS_SCOPE) + + const where = render(dbChainMockFns.where.mock.calls[0]?.[0]) + expect(where.sql).toContain('"document"."acl" && ARRAY[$2, $3]::text[]') + expect(where.params).toEqual(['document-1', 'pub', 'ws']) + }) }) diff --git a/apps/sim/lib/knowledge/chunks/service.ts b/apps/sim/lib/knowledge/chunks/service.ts index 8dd509a5c05..6016be05265 100644 --- a/apps/sim/lib/knowledge/chunks/service.ts +++ b/apps/sim/lib/knowledge/chunks/service.ts @@ -15,6 +15,8 @@ import { textKey, } from '@/lib/api/list-query' import type { DurableSecretProvenance } from '@/lib/execution/durable-secret-provenance' +import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' +import type { KnowledgeAccessScope } from '@/lib/knowledge/access/types' import type { BatchOperationResult, ChunkData, @@ -76,7 +78,8 @@ const CHUNK_SORTS = { export async function queryChunks( documentId: string, filters: ChunkFilters, - requestId: string + requestId: string, + access: KnowledgeAccessScope ): Promise { const { search, @@ -89,7 +92,12 @@ export async function queryChunks( } = filters const keys = CHUNK_SORTS[sortBy] - const conditions = [eq(embedding.documentId, documentId)] + /** + * The document context that reached here was already loaded under the same + * scope; the join repeats the check at the row so a revocation between the + * two reads still hides the content. + */ + const conditions = [eq(embedding.documentId, documentId), knowledgeAccessCondition(access)] if (enabled === 'true') { conditions.push(eq(embedding.enabled, true)) @@ -130,6 +138,7 @@ export async function queryChunks( updatedAt: embedding.updatedAt, }) .from(embedding) + .innerJoin(document, eq(embedding.documentId, document.id)) .where(and(...pageConditions)) .orderBy(...listOrderBy(keysetColumns(keys), sortOrder)) .limit(limit + 1) @@ -138,6 +147,7 @@ export async function queryChunks( const totalCount = await db .select({ count: sql`count(*)` }) .from(embedding) + .innerJoin(document, eq(embedding.documentId, document.id)) .where(and(...conditions)) const page = keysetPage(keys, rows as ChunkData[], limit) diff --git a/apps/sim/lib/knowledge/connectors/member-access.test.ts b/apps/sim/lib/knowledge/connectors/member-access.test.ts new file mode 100644 index 00000000000..276b201ec65 --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/member-access.test.ts @@ -0,0 +1,544 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + requireResourcePolicy: vi.fn(), + writeResourcePolicy: vi.fn(), + loadBinding: vi.fn(), + listOptionCredentials: vi.fn(), + resolveManagedOAuthToken: vi.fn(), + recordAudit: vi.fn(), +})) + +vi.mock('@sim/audit', () => ({ + AuditAction: { + CREDENTIAL_ACCESSED: 'credential.accessed', + CREDENTIAL_GROUP_UPDATED: 'credential_group.updated', + }, + AuditResourceType: { CREDENTIAL: 'credential', CREDENTIAL_GROUP: 'credential_group' }, + recordAudit: mocks.recordAudit, +})) + +vi.mock('@/lib/resource-policies/repository', async () => { + class ResourcePolicyNotFoundError extends Error {} + class ResourcePolicyRevisionConflictError extends Error {} + return { + ResourcePolicyNotFoundError, + ResourcePolicyRevisionConflictError, + requireResourcePolicy: mocks.requireResourcePolicy, + writeResourcePolicy: mocks.writeResourcePolicy, + } +}) + +vi.mock('@/lib/credential-groups/credentials', () => ({ + loadManagedCredentialGroupBinding: mocks.loadBinding, + listCredentialGroupOptionCredentialReferences: mocks.listOptionCredentials, + isManagedCredentialGroupBindingLive: (binding: { + managedOauthStatus: string + enrollmentStatus: string + groupStatus: string + optionStatus: string | null + }) => + binding.managedOauthStatus === 'active' && + (binding.enrollmentStatus === 'in_progress' || binding.enrollmentStatus === 'completed') && + binding.groupStatus === 'active' && + binding.optionStatus === 'active', +})) + +vi.mock('@/lib/credentials/managed-oauth', () => ({ + resolveManagedOAuthToken: mocks.resolveManagedOAuthToken, +})) + +import { compileCredentialGroupWorkflowAccessPolicy } from '@/lib/credential-groups/application/workflow-access-policy' +import { CREDENTIAL_GROUP_KNOWLEDGE_CONNECTOR_ACCESS_LIMIT } from '@/lib/credential-groups/limits' +import { + findListingCapViolation, + grantKnowledgeConnectorCredentialAccess, + KnowledgeConnectorMemberAccessDeniedError, + listKnowledgeConnectorMemberCredentials, + mintKnowledgeConnectorMemberToken, + revokeKnowledgeConnectorCredentialAccess, + validateKnowledgeConnectorMembersBinding, +} from '@/lib/knowledge/connectors/member-access' +import { + ResourcePolicyNotFoundError, + ResourcePolicyRevisionConflictError, +} from '@/lib/resource-policies/repository' + +const GROUP_ID = 'group-1' +const BINDING = { + workspaceId: 'workspace-1', + credentialGroupId: GROUP_ID, + credentialGroupOptionId: 'option-drive', + connectorId: 'connector-1', +} + +function storedPolicy( + revision: number, + knowledgeConnectorAccess: Array<{ credentialGroupOptionId: string; connectorIds: string[] }>, + allowedWorkflowIds: string[] = [] +) { + return { + id: 'policy-1', + workspaceId: 'workspace-1', + revision, + document: compileCredentialGroupWorkflowAccessPolicy({ + credentialGroupId: GROUP_ID, + allowedWorkflowIds, + knowledgeConnectorAccess, + }), + createdAt: new Date(0), + updatedAt: new Date(0), + } +} + +describe('knowledge connector member access', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.writeResourcePolicy.mockImplementation(async (input) => ({ + ...storedPolicy(input.expectedRevision + 1, []), + document: input.document, + })) + }) + + describe('grant', () => { + it('adds the connector under its option while keeping workflow access and audits the group', async () => { + mocks.requireResourcePolicy.mockResolvedValue( + storedPolicy( + 3, + [{ credentialGroupOptionId: 'option-drive', connectorIds: ['connector-0'] }], + ['workflow-1'] + ) + ) + + await grantKnowledgeConnectorCredentialAccess(BINDING, 'admin-1') + + expect(mocks.writeResourcePolicy).toHaveBeenCalledTimes(1) + const written = mocks.writeResourcePolicy.mock.calls[0][0] + expect(written.expectedRevision).toBe(3) + expect(written.actorUserId).toBe('admin-1') + expect( + written.document.statements.map((statement: { sid: string }) => statement.sid) + ).toEqual([ + 'CredentialGroupActorCredentialAccess', + 'WorkflowCredentialAccess', + 'KnowledgeConnectorCredentialAccess:option-drive', + ]) + expect(written.document.statements[2].principals).toEqual([ + { type: 'knowledge_connector', connectorId: 'connector-0' }, + { type: 'knowledge_connector', connectorId: 'connector-1' }, + ]) + expect(mocks.recordAudit).toHaveBeenCalledWith( + expect.objectContaining({ + action: 'credential_group.updated', + actorId: 'admin-1', + resourceId: GROUP_ID, + metadata: expect.objectContaining({ + change: 'granted', + connectorId: 'connector-1', + credentialGroupOptionId: 'option-drive', + revision: 4, + }), + }) + ) + }) + + it('is idempotent when the connector is already bound to that option', async () => { + mocks.requireResourcePolicy.mockResolvedValue( + storedPolicy(3, [ + { credentialGroupOptionId: 'option-drive', connectorIds: ['connector-1'] }, + ]) + ) + + await grantKnowledgeConnectorCredentialAccess(BINDING, 'admin-1') + + expect(mocks.writeResourcePolicy).not.toHaveBeenCalled() + expect(mocks.recordAudit).not.toHaveBeenCalled() + }) + + it('moves a connector between options rather than binding it twice', async () => { + mocks.requireResourcePolicy.mockResolvedValue( + storedPolicy(1, [{ credentialGroupOptionId: 'option-old', connectorIds: ['connector-1'] }]) + ) + + await grantKnowledgeConnectorCredentialAccess(BINDING, 'admin-1') + + const written = mocks.writeResourcePolicy.mock.calls[0][0] + expect( + written.document.statements.map((statement: { sid: string }) => statement.sid) + ).toEqual([ + 'CredentialGroupActorCredentialAccess', + 'KnowledgeConnectorCredentialAccess:option-drive', + ]) + }) + + it('recomputes from the fresh document after a revision conflict', async () => { + mocks.requireResourcePolicy + .mockResolvedValueOnce(storedPolicy(1, [])) + .mockResolvedValueOnce( + storedPolicy(2, [ + { credentialGroupOptionId: 'option-drive', connectorIds: ['connector-9'] }, + ]) + ) + mocks.writeResourcePolicy + .mockRejectedValueOnce(new ResourcePolicyRevisionConflictError()) + .mockImplementationOnce(async (input) => ({ + ...storedPolicy(input.expectedRevision + 1, []), + document: input.document, + })) + + await grantKnowledgeConnectorCredentialAccess(BINDING, 'admin-1') + + expect(mocks.writeResourcePolicy).toHaveBeenCalledTimes(2) + const written = mocks.writeResourcePolicy.mock.calls[1][0] + expect(written.expectedRevision).toBe(2) + expect(written.document.statements[1].principals).toEqual([ + { type: 'knowledge_connector', connectorId: 'connector-1' }, + { type: 'knowledge_connector', connectorId: 'connector-9' }, + ]) + }) + + it('gives up as a conflict when the policy keeps changing', async () => { + mocks.requireResourcePolicy.mockResolvedValue(storedPolicy(1, [])) + mocks.writeResourcePolicy.mockRejectedValue(new ResourcePolicyRevisionConflictError()) + + await expect( + grantKnowledgeConnectorCredentialAccess(BINDING, 'admin-1') + ).rejects.toMatchObject({ code: 'conflict' }) + }) + + it('refuses to bind more connectors than one option may back', async () => { + mocks.requireResourcePolicy.mockResolvedValue( + storedPolicy(1, [ + { + credentialGroupOptionId: 'option-drive', + connectorIds: Array.from( + { length: CREDENTIAL_GROUP_KNOWLEDGE_CONNECTOR_ACCESS_LIMIT }, + (_, index) => `connector-${String(index).padStart(3, '0')}` + ), + }, + ]) + ) + + await expect( + grantKnowledgeConnectorCredentialAccess( + { ...BINDING, connectorId: 'connector-new' }, + 'admin-1' + ) + ).rejects.toMatchObject({ code: 'validation' }) + expect(mocks.writeResourcePolicy).not.toHaveBeenCalled() + }) + }) + + describe('revoke', () => { + it('removes the connector and drops an emptied option statement', async () => { + mocks.requireResourcePolicy.mockResolvedValue( + storedPolicy(5, [ + { credentialGroupOptionId: 'option-drive', connectorIds: ['connector-1'] }, + { credentialGroupOptionId: 'option-other', connectorIds: ['connector-2'] }, + ]) + ) + + await revokeKnowledgeConnectorCredentialAccess(BINDING, 'admin-1') + + const written = mocks.writeResourcePolicy.mock.calls[0][0] + expect( + written.document.statements.map((statement: { sid: string }) => statement.sid) + ).toEqual([ + 'CredentialGroupActorCredentialAccess', + 'KnowledgeConnectorCredentialAccess:option-other', + ]) + expect(mocks.recordAudit).toHaveBeenCalledWith( + expect.objectContaining({ metadata: expect.objectContaining({ change: 'revoked' }) }) + ) + }) + + it('is a no-op when the connector was never bound', async () => { + mocks.requireResourcePolicy.mockResolvedValue(storedPolicy(5, [])) + + await revokeKnowledgeConnectorCredentialAccess(BINDING, 'admin-1') + + expect(mocks.writeResourcePolicy).not.toHaveBeenCalled() + }) + + it('tolerates a group whose policy is already gone', async () => { + mocks.requireResourcePolicy.mockRejectedValue(new ResourcePolicyNotFoundError()) + + await expect( + revokeKnowledgeConnectorCredentialAccess(BINDING, 'admin-1') + ).resolves.toBeUndefined() + }) + }) + + describe('mint', () => { + const mintInput = { + connectorId: 'connector-1', + workspaceId: 'workspace-1', + credentialId: 'credential-1', + expectedProviderId: 'google-drive', + requiredScopes: ['https://www.googleapis.com/auth/drive'], + runId: 'run-1', + } + + beforeEach(() => { + mocks.loadBinding.mockResolvedValue({ + credentialId: 'credential-1', + workspaceId: 'workspace-1', + providerId: 'google-drive', + credentialGroupId: GROUP_ID, + credentialGroupOptionId: 'option-drive', + managedOauthStatus: 'active', + enrollmentStatus: 'completed', + groupStatus: 'active', + optionStatus: 'active', + }) + mocks.resolveManagedOAuthToken.mockResolvedValue({ accessToken: 'token', refreshed: false }) + }) + + it('resolves the token when the policy names the connector under the credential option and audits it', async () => { + mocks.requireResourcePolicy.mockResolvedValue( + storedPolicy(2, [ + { credentialGroupOptionId: 'option-drive', connectorIds: ['connector-1'] }, + ]) + ) + + await expect(mintKnowledgeConnectorMemberToken(mintInput)).resolves.toEqual({ + accessToken: 'token', + refreshed: false, + }) + + expect(mocks.resolveManagedOAuthToken).toHaveBeenCalledWith({ + credentialId: 'credential-1', + workspaceId: 'workspace-1', + expectedProviderId: 'google-drive', + requiredScopes: ['https://www.googleapis.com/auth/drive'], + }) + expect(mocks.recordAudit).toHaveBeenCalledWith( + expect.objectContaining({ + action: 'credential.accessed', + actorId: null, + resourceId: 'credential-1', + metadata: expect.objectContaining({ + connectorId: 'connector-1', + credentialGroupOptionId: 'option-drive', + runId: 'run-1', + }), + }) + ) + }) + + it('denies a connector the policy does not name', async () => { + mocks.requireResourcePolicy.mockResolvedValue( + storedPolicy(2, [ + { credentialGroupOptionId: 'option-drive', connectorIds: ['connector-2'] }, + ]) + ) + + await expect(mintKnowledgeConnectorMemberToken(mintInput)).rejects.toBeInstanceOf( + KnowledgeConnectorMemberAccessDeniedError + ) + expect(mocks.resolveManagedOAuthToken).not.toHaveBeenCalled() + expect(mocks.recordAudit).not.toHaveBeenCalled() + }) + + it('denies a credential collected under a different option', async () => { + mocks.requireResourcePolicy.mockResolvedValue( + storedPolicy(2, [ + { credentialGroupOptionId: 'option-other', connectorIds: ['connector-1'] }, + ]) + ) + + await expect(mintKnowledgeConnectorMemberToken(mintInput)).rejects.toBeInstanceOf( + KnowledgeConnectorMemberAccessDeniedError + ) + expect(mocks.resolveManagedOAuthToken).not.toHaveBeenCalled() + }) + + it.each([ + ['a revoked enrollment', { enrollmentStatus: 'revoked' }], + ['a disabled option', { optionStatus: 'disabled' }], + ['a removed option', { optionStatus: null }], + ['a disabled group', { groupStatus: 'disabled' }], + ['a credential needing re-auth', { managedOauthStatus: 'needs_reauth' }], + ] as const)('denies %s before consulting any policy', async (_name, overrides) => { + mocks.loadBinding.mockResolvedValue({ + credentialId: 'credential-1', + workspaceId: 'workspace-1', + providerId: 'google-drive', + credentialGroupId: GROUP_ID, + credentialGroupOptionId: 'option-drive', + managedOauthStatus: 'active', + enrollmentStatus: 'completed', + groupStatus: 'active', + optionStatus: 'active', + ...overrides, + }) + + await expect(mintKnowledgeConnectorMemberToken(mintInput)).rejects.toBeInstanceOf( + KnowledgeConnectorMemberAccessDeniedError + ) + expect(mocks.requireResourcePolicy).not.toHaveBeenCalled() + }) + + it('denies a credential from another workspace before consulting any policy', async () => { + mocks.loadBinding.mockResolvedValue({ + credentialId: 'credential-1', + workspaceId: 'workspace-2', + providerId: 'google-drive', + credentialGroupId: GROUP_ID, + credentialGroupOptionId: 'option-drive', + managedOauthStatus: 'active', + enrollmentStatus: 'completed', + groupStatus: 'active', + optionStatus: 'active', + }) + + await expect(mintKnowledgeConnectorMemberToken(mintInput)).rejects.toBeInstanceOf( + KnowledgeConnectorMemberAccessDeniedError + ) + expect(mocks.requireResourcePolicy).not.toHaveBeenCalled() + }) + + it('denies when the group policy no longer exists', async () => { + mocks.requireResourcePolicy.mockRejectedValue(new ResourcePolicyNotFoundError()) + + await expect(mintKnowledgeConnectorMemberToken(mintInput)).rejects.toBeInstanceOf( + KnowledgeConnectorMemberAccessDeniedError + ) + }) + }) + + describe('list', () => { + it('pages the option credentials only for a granted connector', async () => { + mocks.requireResourcePolicy.mockResolvedValue( + storedPolicy(2, [ + { credentialGroupOptionId: 'option-drive', connectorIds: ['connector-1'] }, + ]) + ) + mocks.listOptionCredentials.mockResolvedValue({ credentials: [], nextCursor: null }) + + await listKnowledgeConnectorMemberCredentials({ ...BINDING, limit: 50, cursor: 'c-1' }) + + expect(mocks.listOptionCredentials).toHaveBeenCalledWith({ + workspaceId: 'workspace-1', + credentialGroupId: GROUP_ID, + credentialGroupOptionId: 'option-drive', + limit: 50, + cursor: 'c-1', + }) + }) + + it('refuses to enumerate members for a connector without a grant', async () => { + mocks.requireResourcePolicy.mockResolvedValue(storedPolicy(2, [])) + + await expect( + listKnowledgeConnectorMemberCredentials({ ...BINDING, limit: 50 }) + ).rejects.toBeInstanceOf(KnowledgeConnectorMemberAccessDeniedError) + expect(mocks.listOptionCredentials).not.toHaveBeenCalled() + }) + }) + + describe('binding validation', () => { + const driveMeta = { + name: 'Google Drive', + auth: { + mode: 'oauth' as const, + provider: 'google-drive' as const, + requiredScopes: ['https://www.googleapis.com/auth/drive'], + }, + permissionScopedListing: { capFieldIds: ['maxFiles'] }, + configFields: [{ id: 'maxFiles', title: 'Max Files', type: 'short-input' as const }], + } + const driveOption = { + id: 'option-drive', + provider: 'google-drive', + label: 'Drive', + authorizationAppId: 'google:app', + requiredScopes: [ + 'openid', + 'https://www.googleapis.com/auth/userinfo.email', + 'https://www.googleapis.com/auth/userinfo.profile', + 'https://www.googleapis.com/auth/drive.file', + 'https://www.googleapis.com/auth/drive', + ], + scopeVersion: 1, + required: true, + status: 'active' as const, + } + const group = { status: 'active' as const, options: [driveOption] } + + it('accepts a matching, fully scoped, uncapped binding', () => { + expect( + validateKnowledgeConnectorMembersBinding({ + connectorMeta: driveMeta, + group, + credentialGroupOptionId: 'option-drive', + sourceConfig: { folderId: ['folder-1'], maxFiles: '' }, + }) + ).toEqual({ ok: true, option: driveOption }) + }) + + it.each([ + [ + 'a connector whose listing is not permission scoped', + { connectorMeta: { ...driveMeta, permissionScopedListing: undefined } }, + 'cannot sync per member', + ], + ['a disabled group', { group: { ...group, status: 'disabled' as const } }, 'is disabled'], + ['an unknown option', { credentialGroupOptionId: 'option-missing' }, 'was not found'], + [ + 'a disabled option', + { group: { ...group, options: [{ ...driveOption, status: 'disabled' as const }] } }, + 'option is disabled', + ], + [ + 'an option for another provider', + { group: { ...group, options: [{ ...driveOption, provider: 'google-calendar' }] } }, + 'needs google-drive', + ], + [ + 'an option missing a required scope', + { + group: { + ...group, + options: [ + { + ...driveOption, + requiredScopes: ['openid', 'https://www.googleapis.com/auth/drive.file'], + }, + ], + }, + }, + 'every permission', + ], + ['a listing cap', { sourceConfig: { maxFiles: '500' } }, 'Max Files cannot be set'], + ])('rejects %s', (_name, overrides, message) => { + const result = validateKnowledgeConnectorMembersBinding({ + connectorMeta: driveMeta, + group, + credentialGroupOptionId: 'option-drive', + sourceConfig: {}, + ...overrides, + }) + expect(result.ok).toBe(false) + if (!result.ok) expect(result.message).toContain(message) + }) + }) +}) + +describe('findListingCapViolation', () => { + const meta = { + permissionScopedListing: { capFieldIds: ['maxFiles'] }, + configFields: [{ id: 'maxFiles', title: 'Max Files' }], + } as never + + it.each([[undefined], [null], [''], ['0'], [0], [' 0 ']])('treats %j as unlimited', (value) => { + expect(findListingCapViolation(meta, { maxFiles: value })).toBeNull() + }) + + it.each([['5'], [5], ['abc']])('refuses %j', (value) => { + expect(findListingCapViolation(meta, { maxFiles: value })).toContain('Max Files') + }) +}) diff --git a/apps/sim/lib/knowledge/connectors/member-access.ts b/apps/sim/lib/knowledge/connectors/member-access.ts new file mode 100644 index 00000000000..4dcae8586fc --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/member-access.ts @@ -0,0 +1,474 @@ +import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' +import type { CredentialGroupOptionConfig } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { getManagedOAuthConnectorPolicy } from '@/lib/auth/connectors/managed-oauth' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { + type CredentialGroupKnowledgeConnectorAccess, + compileCredentialGroupWorkflowAccessPolicy, + credentialGroupWorkflowAccessPolicyCodec, + decodeCredentialGroupKnowledgeConnectorAccess, + decodeCredentialGroupWorkflowAccessPolicy, + evaluateCredentialGroupKnowledgeConnectorAccess, +} from '@/lib/credential-groups/application/workflow-access-policy' +import { + type CredentialGroupCredentialListContext, + type CredentialGroupOptionCredentialReference, + isManagedCredentialGroupBindingLive, + listCredentialGroupOptionCredentialReferences, + loadManagedCredentialGroupBinding, +} from '@/lib/credential-groups/credentials' +import { CREDENTIAL_GROUP_KNOWLEDGE_CONNECTOR_ACCESS_LIMIT } from '@/lib/credential-groups/limits' +import { + getCredentialGroupProviderId, + isCredentialGroupProvider, +} from '@/lib/credential-groups/providers' +import { + type ResolvedManagedOAuthToken, + resolveManagedOAuthToken, +} from '@/lib/credentials/managed-oauth' +import { + CREDENTIAL_GROUP_CREDENTIAL_USE_ACTION, + type ResourcePolicyBindingFor, +} from '@/lib/resource-policies/registry' +import { + ResourcePolicyNotFoundError, + ResourcePolicyRevisionConflictError, + requireResourcePolicy, + writeResourcePolicy, +} from '@/lib/resource-policies/repository' +import type { ConnectorMeta } from '@/connectors/types' + +const logger = createLogger('KnowledgeConnectorMemberAccess') + +/** Concurrent editors of one group's policy are rare; a handful of retries absorbs them. */ +const POLICY_WRITE_ATTEMPTS = 5 + +const CREDENTIAL_USE_BINDING = { + resourceType: 'credential_group', + action: CREDENTIAL_GROUP_CREDENTIAL_USE_ACTION, +} as const satisfies ResourcePolicyBindingFor<'credential_group'> + +/** Raised when the group's policy does not let this connector use the credential. */ +export class KnowledgeConnectorMemberAccessDeniedError extends Error { + constructor(message: string) { + super(message) + this.name = 'KnowledgeConnectorMemberAccessDeniedError' + } +} + +/** The credential-group slot a members-mode connector crawls with. */ +export interface KnowledgeConnectorCredentialBinding { + workspaceId: string + credentialGroupId: string + credentialGroupOptionId: string + connectorId: string +} + +function policyTarget( + binding: Pick +) { + return { + workspaceId: binding.workspaceId, + resourceType: 'credential_group' as const, + resourceId: binding.credentialGroupId, + codec: credentialGroupWorkflowAccessPolicyCodec, + } +} + +function withoutConnector( + access: readonly CredentialGroupKnowledgeConnectorAccess[], + connectorId: string +): CredentialGroupKnowledgeConnectorAccess[] { + return access.map((entry) => ({ + credentialGroupOptionId: entry.credentialGroupOptionId, + connectorIds: entry.connectorIds.filter((id) => id !== connectorId), + })) +} + +function withConnector( + access: readonly CredentialGroupKnowledgeConnectorAccess[], + binding: Pick +): CredentialGroupKnowledgeConnectorAccess[] { + const next = withoutConnector(access, binding.connectorId) + const option = next.find( + (entry) => entry.credentialGroupOptionId === binding.credentialGroupOptionId + ) + if (option) { + if (option.connectorIds.length >= CREDENTIAL_GROUP_KNOWLEDGE_CONNECTOR_ACCESS_LIMIT) { + throw new OrchestrationError( + 'validation', + `A credential option can back at most ${CREDENTIAL_GROUP_KNOWLEDGE_CONNECTOR_ACCESS_LIMIT} knowledge connectors` + ) + } + option.connectorIds.push(binding.connectorId) + return next + } + return [ + ...next, + { + credentialGroupOptionId: binding.credentialGroupOptionId, + connectorIds: [binding.connectorId], + }, + ] +} + +function sameAccess( + left: readonly CredentialGroupKnowledgeConnectorAccess[], + right: readonly CredentialGroupKnowledgeConnectorAccess[] +): boolean { + const normalise = (access: readonly CredentialGroupKnowledgeConnectorAccess[]) => + JSON.stringify( + access + .filter((entry) => entry.connectorIds.length > 0) + .map((entry) => [entry.credentialGroupOptionId, [...entry.connectorIds].sort()]) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + ) + return normalise(left) === normalise(right) +} + +/** + * Rewrites the group's knowledge connector statements through the policy's + * revision CAS. A concurrent edit (an admin changing workflow access, another + * connector being bound) loses the race, and the rewrite is recomputed from the + * fresh document rather than retried blindly, so nobody's statement is lost. + */ +async function rewriteKnowledgeConnectorAccess(input: { + binding: Pick + actorUserId: string + change: 'granted' | 'revoked' + connectorId: string + credentialGroupOptionId?: string + mutate: ( + access: readonly CredentialGroupKnowledgeConnectorAccess[] + ) => CredentialGroupKnowledgeConnectorAccess[] +}): Promise { + const target = policyTarget(input.binding) + for (let attempt = 1; attempt <= POLICY_WRITE_ATTEMPTS; attempt += 1) { + const existing = await requireResourcePolicy(target) + const current = decodeCredentialGroupKnowledgeConnectorAccess( + existing.document, + input.binding.credentialGroupId + ) + const next = input.mutate(current) + if (sameAccess(current, next)) return + + const document = compileCredentialGroupWorkflowAccessPolicy({ + credentialGroupId: input.binding.credentialGroupId, + allowedWorkflowIds: decodeCredentialGroupWorkflowAccessPolicy( + existing.document, + input.binding.credentialGroupId + ), + knowledgeConnectorAccess: next, + }) + try { + const written = await writeResourcePolicy({ + ...target, + expectedRevision: existing.revision, + actorUserId: input.actorUserId, + document, + }) + recordAudit({ + workspaceId: input.binding.workspaceId, + actorId: input.actorUserId, + action: AuditAction.CREDENTIAL_GROUP_UPDATED, + resourceType: AuditResourceType.CREDENTIAL_GROUP, + resourceId: input.binding.credentialGroupId, + description: + input.change === 'granted' + ? 'Granted a knowledge connector access to Credential Group credentials' + : 'Revoked a knowledge connector’s access to Credential Group credentials', + metadata: { + revision: written.revision, + change: input.change, + connectorId: input.connectorId, + ...(input.credentialGroupOptionId + ? { credentialGroupOptionId: input.credentialGroupOptionId } + : {}), + }, + }) + return + } catch (error) { + if (!(error instanceof ResourcePolicyRevisionConflictError)) throw error + logger.info('Credential Group policy changed while binding a knowledge connector; retrying', { + credentialGroupId: input.binding.credentialGroupId, + connectorId: input.connectorId, + attempt, + }) + } + } + throw new OrchestrationError( + 'conflict', + 'Credential Group access is being edited by someone else. Try again.' + ) +} + +/** + * Lets the connector use credentials collected under one option. Moving a + * connector between options replaces its previous binding, since a connector + * crawls with exactly one credential slot. + */ +export async function grantKnowledgeConnectorCredentialAccess( + binding: KnowledgeConnectorCredentialBinding, + actorUserId: string +): Promise { + await rewriteKnowledgeConnectorAccess({ + binding, + actorUserId, + change: 'granted', + connectorId: binding.connectorId, + credentialGroupOptionId: binding.credentialGroupOptionId, + mutate: (access) => withConnector(access, binding), + }) +} + +/** + * Removes the connector from every option it was bound to. A group whose policy + * is already gone (the group was deleted, which cascades the policy) has nothing + * left to revoke. + */ +export async function revokeKnowledgeConnectorCredentialAccess( + binding: Pick< + KnowledgeConnectorCredentialBinding, + 'workspaceId' | 'credentialGroupId' | 'connectorId' + >, + actorUserId: string +): Promise { + try { + await rewriteKnowledgeConnectorAccess({ + binding, + actorUserId, + change: 'revoked', + connectorId: binding.connectorId, + mutate: (access) => withoutConnector(access, binding.connectorId), + }) + } catch (error) { + if (error instanceof ResourcePolicyNotFoundError) return + throw error + } +} + +/** Throws unless the group's policy lets this connector use the option's credentials. */ +export async function assertKnowledgeConnectorCredentialAccess( + binding: KnowledgeConnectorCredentialBinding +): Promise { + const policy = await requireResourcePolicy(policyTarget(binding)).catch((error: unknown) => { + if (error instanceof ResourcePolicyNotFoundError) { + throw new KnowledgeConnectorMemberAccessDeniedError( + 'Credential Group no longer has an access policy' + ) + } + throw error + }) + const decision = evaluateCredentialGroupKnowledgeConnectorAccess({ + document: policy.document, + credentialGroupId: binding.credentialGroupId, + connectorId: binding.connectorId, + credentialGroupOptionId: binding.credentialGroupOptionId, + resourcePolicy: CREDENTIAL_USE_BINDING, + }) + if (decision.decision !== 'allow') { + throw new KnowledgeConnectorMemberAccessDeniedError( + 'Credential Group policy does not grant this knowledge connector access to the credential option' + ) + } +} + +export interface MintKnowledgeConnectorMemberTokenInput { + connectorId: string + workspaceId: string + credentialId: string + expectedProviderId: string + requiredScopes: string[] + /** The sync run the access is attributed to in the audit trail. */ + runId: string +} + +/** + * Resolves a member's managed token for a crawl. Not a workspace use case: the + * sync job has no principal, so the group's own policy is the whole + * authorization — the connector must be named under the credential's option. + * Every mint is audited against the credential, attributed to the run. + */ +export async function mintKnowledgeConnectorMemberToken( + input: MintKnowledgeConnectorMemberTokenInput +): Promise { + const binding = await loadManagedCredentialGroupBinding(input.credentialId) + if (!binding || binding.workspaceId !== input.workspaceId) { + throw new KnowledgeConnectorMemberAccessDeniedError( + 'Managed credential is not enrolled in a Credential Group in this workspace' + ) + } + if (!isManagedCredentialGroupBindingLive(binding)) { + throw new KnowledgeConnectorMemberAccessDeniedError( + 'Managed credential is not currently usable: its enrollment, option, or group is not active' + ) + } + await assertKnowledgeConnectorCredentialAccess({ + workspaceId: binding.workspaceId, + credentialGroupId: binding.credentialGroupId, + credentialGroupOptionId: binding.credentialGroupOptionId, + connectorId: input.connectorId, + }) + const token = await resolveManagedOAuthToken({ + credentialId: binding.credentialId, + workspaceId: binding.workspaceId, + expectedProviderId: input.expectedProviderId, + requiredScopes: input.requiredScopes, + }) + recordAudit({ + workspaceId: binding.workspaceId, + actorId: null, + actorName: 'Knowledge connector sync', + action: AuditAction.CREDENTIAL_ACCESSED, + resourceType: AuditResourceType.CREDENTIAL, + resourceId: binding.credentialId, + description: `Accessed managed OAuth credential for provider ${input.expectedProviderId} on behalf of a knowledge connector`, + metadata: { + provider: input.expectedProviderId, + credentialType: 'managed_oauth', + connectorId: input.connectorId, + credentialGroupId: binding.credentialGroupId, + credentialGroupOptionId: binding.credentialGroupOptionId, + runId: input.runId, + }, + }) + return token +} + +export interface ListKnowledgeConnectorMemberCredentialsInput + extends KnowledgeConnectorCredentialBinding { + limit: number + cursor?: string +} + +/** + * Pages every credential collected under the connector's option, in any state, + * after proving the connector is granted that option. Membership reconciliation + * needs the unusable credentials too — they are what gets suspended. + */ +export async function listKnowledgeConnectorMemberCredentials( + input: ListKnowledgeConnectorMemberCredentialsInput +): Promise<{ credentials: CredentialGroupOptionCredentialReference[]; nextCursor: string | null }> { + await assertKnowledgeConnectorCredentialAccess(input) + return listCredentialGroupOptionCredentialReferences({ + workspaceId: input.workspaceId, + credentialGroupId: input.credentialGroupId, + credentialGroupOptionId: input.credentialGroupOptionId, + limit: input.limit, + cursor: input.cursor, + }) +} + +export type KnowledgeConnectorMembersBindingValidation = + | { ok: true; option: CredentialGroupOptionConfig } + | { ok: false; message: string } + +/** Whether a listing cap is in force. Blank, `0`, and `'0'` all mean unlimited. */ +function isCapFieldSet(value: unknown): boolean { + if (value === undefined || value === null) return false + if (typeof value === 'number') return value > 0 + if (typeof value === 'string') { + const trimmed = value.trim() + if (trimmed.length === 0) return false + const parsed = Number(trimmed) + return !(Number.isFinite(parsed) && parsed <= 0) + } + return true +} + +/** + * The source config with its listing caps set to 0, which every connector + * reads as unlimited (an absent cap falls back to a connector's default). A + * cap has no meaning once a connector syncs per member, so the switch clears it rather than refusing a + * connector the admin can no longer see the field on. + */ +export function stripListingCapFields( + connectorMeta: Pick, + sourceConfig: Record +): Record { + const capFieldIds = connectorMeta.permissionScopedListing?.capFieldIds ?? [] + if (capFieldIds.length === 0) return sourceConfig + const stripped = { ...sourceConfig } + for (const fieldId of capFieldIds) stripped[fieldId] = 0 + return stripped +} + +/** + * The message refusing a source config that caps a per-member listing, or + * null when nothing caps it. A cap would hide part of a member's corpus and + * suppress removals forever, so it is refused on every members-mode save. + */ +export function findListingCapViolation( + connectorMeta: Pick, + sourceConfig: Record +): string | null { + const capFields = (connectorMeta.permissionScopedListing?.capFieldIds ?? []).filter((fieldId) => + isCapFieldSet(sourceConfig[fieldId]) + ) + if (capFields.length === 0) return null + const titles = capFields.map( + (fieldId) => connectorMeta.configFields.find((field) => field.id === fieldId)?.title ?? fieldId + ) + return `${titles.join(', ')} cannot be set when syncing per member: every member's listing must be complete for their access to be tracked` +} + +/** + * Decides whether a connector may crawl per member with one option's + * credentials. Pure apart from the provider's own scope policy, so a route can + * refuse a binding before any credential is touched. + */ +export function validateKnowledgeConnectorMembersBinding(input: { + connectorMeta: Pick + group: Pick + credentialGroupOptionId: string + sourceConfig: Record +}): KnowledgeConnectorMembersBindingValidation { + const { connectorMeta, group } = input + if (!connectorMeta.permissionScopedListing) { + return { + ok: false, + message: `${connectorMeta.name} cannot sync per member: its listing does not reflect who may read each document`, + } + } + if (connectorMeta.auth.mode !== 'oauth') { + return { ok: false, message: `${connectorMeta.name} does not authenticate with OAuth` } + } + if (group.status !== 'active') { + return { ok: false, message: 'Credential Group is disabled' } + } + const option = group.options.find((candidate) => candidate.id === input.credentialGroupOptionId) + if (!option) { + return { ok: false, message: 'Credential option was not found in this Credential Group' } + } + if (option.status !== 'active') { + return { ok: false, message: 'Credential option is disabled' } + } + if ( + !isCredentialGroupProvider(option.provider) || + getCredentialGroupProviderId(option.provider) !== connectorMeta.auth.provider + ) { + return { + ok: false, + message: `Credential option collects ${option.provider} accounts, but ${connectorMeta.name} needs ${connectorMeta.auth.provider}`, + } + } + const scopePolicy = getManagedOAuthConnectorPolicy(connectorMeta.auth.provider) + if (!scopePolicy) { + return { + ok: false, + message: `${connectorMeta.auth.provider} is not a managed OAuth provider`, + } + } + if ( + !scopePolicy.hasRequiredScopes(option.requiredScopes, connectorMeta.auth.requiredScopes ?? []) + ) { + return { + ok: false, + message: `Credential option does not request every permission ${connectorMeta.name} needs to read the source`, + } + } + const capViolation = findListingCapViolation(connectorMeta, input.sourceConfig) + if (capViolation) return { ok: false, message: capViolation } + return { ok: true, option } +} diff --git a/apps/sim/lib/knowledge/connectors/member-observations.test.ts b/apps/sim/lib/knowledge/connectors/member-observations.test.ts new file mode 100644 index 00000000000..5c34c588f33 --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/member-observations.test.ts @@ -0,0 +1,157 @@ +/** + * @vitest-environment node + */ +import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/knowledge/documents/service', () => ({ + ConnectorSyncDeletionGuardError: class ConnectorSyncDeletionGuardError extends Error {}, + hardDeleteDocuments: vi.fn(), +})) + +import { db } from '@sim/db' +import { + applyMemberDocumentLifecycle, + rewriteConnectorAcls, + staleMemberWindowMs, + sweepStaleMemberObservations, +} from '@/lib/knowledge/connectors/member-observations' +import { MEMBER_OBSERVATION_STALE_AFTER_HOURS } from '@/lib/knowledge/connectors/sync-limits' +import { SyncLockLostException } from '@/lib/knowledge/connectors/sync-lock' +import { + ConnectorSyncDeletionGuardError, + hardDeleteDocuments, +} from '@/lib/knowledge/documents/service' + +const NOW = new Date('2026-09-01T12:00:00Z') +const STALE_MEMBER = { id: 'm-1', connectorId: 'c-1', syncIntervalMinutes: 60 } + +describe('staleMemberWindowMs', () => { + it('is the larger of a day and two intervals', () => { + const day = MEMBER_OBSERVATION_STALE_AFTER_HOURS * 60 * 60 * 1000 + expect(staleMemberWindowMs(60)).toBe(day) + expect(staleMemberWindowMs(0)).toBe(day) + expect(staleMemberWindowMs(24 * 60)).toBe(2 * 24 * 60 * 60 * 1000) + }) +}) + +describe('sweepStaleMemberObservations', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + it('sweeps a member that is still stale once its row is locked', async () => { + queueTableRows(schemaMock.knowledgeConnectorMember, [STALE_MEMBER]) + queueTableRows(schemaMock.knowledgeConnector, [{ id: 'c-1' }]) + queueTableRows(schemaMock.knowledgeConnectorMember, [{ id: 'm-1' }]) + dbChainMockFns.returning + .mockResolvedValueOnce([{ documentId: 'd-1' }, { documentId: 'd-2' }]) + .mockResolvedValueOnce([{ id: 'd-1' }, { id: 'd-2' }]) + .mockResolvedValueOnce([{ id: 'd-2' }]) + + await expect(sweepStaleMemberObservations(NOW)).resolves.toEqual({ + members: 1, + observationsRemoved: 2, + documentsRematerialized: 2, + docsTombstoned: 1, + }) + + expect(dbChainMockFns.transaction).toHaveBeenCalledOnce() + expect(dbChainMockFns.for).toHaveBeenNthCalledWith(1, 'share') + expect(dbChainMockFns.for).toHaveBeenNthCalledWith(2, 'update') + expect(dbChainMockFns.delete).toHaveBeenCalledWith(schemaMock.knowledgeDocumentObservation) + expect(dbChainMockFns.set).toHaveBeenLastCalledWith({ deletedAt: NOW }) + }) + + /** + * A run that claimed the member between the selection and the lock moved + * `lastStartedAt` forward, so the re-check under `FOR UPDATE` finds nothing + * and the observations that run is about to write are left alone. + */ + it('leaves a member that a run claimed after it was selected', async () => { + queueTableRows(schemaMock.knowledgeConnectorMember, [STALE_MEMBER]) + queueTableRows(schemaMock.knowledgeConnector, [{ id: 'c-1' }]) + queueTableRows(schemaMock.knowledgeConnectorMember, []) + + await expect(sweepStaleMemberObservations(NOW)).resolves.toEqual({ + members: 0, + observationsRemoved: 0, + documentsRematerialized: 0, + docsTombstoned: 0, + }) + + expect(dbChainMockFns.delete).not.toHaveBeenCalled() + expect(dbChainMockFns.update).not.toHaveBeenCalled() + }) + + /** A connector that left members mode after the selection no longer matches the shared lock's re-check. */ + it('leaves a connector that left members mode after it was selected', async () => { + queueTableRows(schemaMock.knowledgeConnectorMember, [STALE_MEMBER]) + queueTableRows(schemaMock.knowledgeConnector, []) + + await expect(sweepStaleMemberObservations(NOW)).resolves.toMatchObject({ members: 0 }) + + expect(dbChainMockFns.for).toHaveBeenCalledWith('share') + expect(dbChainMockFns.for).not.toHaveBeenCalledWith('update') + expect(dbChainMockFns.delete).not.toHaveBeenCalled() + }) +}) + +describe('rewriteConnectorAcls', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + }) + + it('proves the lease inside each batch transaction before rewriting', async () => { + queueTableRows(schemaMock.knowledgeConnector, [{ id: 'c-1' }]) + dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'd-1' }]) + + await expect( + rewriteConnectorAcls('c-1', [], { lease: { stillHeld: () => 'held' as never } }) + ).resolves.toBe(true) + + expect(dbChainMockFns.transaction).toHaveBeenCalledOnce() + expect(dbChainMockFns.for).toHaveBeenCalledWith('share') + expect(dbChainMockFns.update).toHaveBeenCalledWith(schemaMock.document) + }) + + it('stops without writing once the lease is gone', async () => { + queueTableRows(schemaMock.knowledgeConnector, []) + + await expect( + rewriteConnectorAcls('c-1', [], { lease: { stillHeld: () => 'lost' as never } }) + ).rejects.toBeInstanceOf(SyncLockLostException) + + expect(dbChainMockFns.update).not.toHaveBeenCalled() + }) +}) + +describe('applyMemberDocumentLifecycle', () => { + beforeEach(() => { + resetDbChainMock() + vi.mocked(hardDeleteDocuments).mockReset() + }) + + it('reports a reclaimed lease during a purge batch as the run being superseded', async () => { + dbChainMockFns.returning.mockResolvedValueOnce([]) + queueTableRows(schemaMock.document, []) + queueTableRows(schemaMock.document, [{ id: 'd-1' }]) + vi.mocked(hardDeleteDocuments).mockRejectedValueOnce( + new ConnectorSyncDeletionGuardError('lease reclaimed') + ) + + await expect( + applyMemberDocumentLifecycle({ + connectorId: 'c-1', + knowledgeBaseId: 'kb-1', + runId: 'run-1', + withLease: (fn) => fn(db as never), + failedExternalIds: new Set(), + allowRemoval: true, + lease: { beatIfDue: async () => {} } as never, + }) + ).rejects.toBeInstanceOf(SyncLockLostException) + }) +}) diff --git a/apps/sim/lib/knowledge/connectors/member-observations.ts b/apps/sim/lib/knowledge/connectors/member-observations.ts new file mode 100644 index 00000000000..361db08c282 --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/member-observations.ts @@ -0,0 +1,558 @@ +import { db } from '@sim/db' +import { + document, + knowledgeConnector, + knowledgeConnectorMember, + knowledgeDocumentObservation, +} from '@sim/db/schema' +import { + and, + eq, + exists, + gt, + inArray, + isNotNull, + isNull, + lt, + ne, + notExists, + or, + sql, +} from 'drizzle-orm' +import type { DbOrTx } from '@/lib/db/types' +import { textArrayLiteral } from '@/lib/knowledge/access/predicate' +import { + MEMBER_OBSERVATION_STALE_AFTER_HOURS, + MEMBER_PURGE_MAX_PER_RUN, + MEMBER_TOMBSTONE_PURGE_DAYS, +} from '@/lib/knowledge/connectors/sync-limits' +import { + assertSyncLeaseHeldInTx, + connectorIsLive, + MEMBER_LOCKABLE_CONNECTOR_STATUSES, + SyncLockLostException, + type SyncRunLease, + type SyncWriteLease, +} from '@/lib/knowledge/connectors/sync-lock' +import { + type ConnectorSyncDeletionGuard, + ConnectorSyncDeletionGuardError, + hardDeleteDocuments, +} from '@/lib/knowledge/documents/service' + +/** Documents rematerialised per `UPDATE`; keeps each statement's bind list and lock footprint small. */ +const MATERIALIZE_BATCH_SIZE = 500 +/** Observation rows written per `INSERT`. */ +const OBSERVATION_BATCH_SIZE = 500 +/** Documents hard-deleted per call, so the lease heartbeat runs between chunks. */ +const PURGE_CHUNK_SIZE = 25 +/** Members one scheduler tick will sweep; the rest wait for the next tick. */ +const STALE_MEMBER_SWEEP_LIMIT = 200 + +/** + * The subject-token aggregate that is a members-mode document's ACL. Ordered + * under the "C" collation so the array matches the code-unit order every other + * writer of an ACL produces. + */ +function observedAcl() { + return sql`COALESCE(( + SELECT array_agg(${knowledgeConnectorMember.subjectToken} ORDER BY ${knowledgeConnectorMember.subjectToken} COLLATE "C") + FROM ${knowledgeDocumentObservation} + JOIN ${knowledgeConnectorMember} + ON ${knowledgeConnectorMember.id} = ${knowledgeDocumentObservation.memberId} + AND ${knowledgeConnectorMember.status} = 'active' + WHERE ${knowledgeDocumentObservation.documentId} = ${document.id} + ), '{}'::text[])` +} + +function observationQuery() { + return db + .select({ one: sql`1` }) + .from(knowledgeDocumentObservation) + .where(eq(knowledgeDocumentObservation.documentId, document.id)) +} + +function hasNoObservation() { + return notExists(observationQuery()) +} + +function hasObservation() { + return exists(observationQuery()) +} + +/** + * Asserts "member M's crawl returned these documents" for this run. Rows that + * already existed keep their identity and move to this run; the count of rows + * that did not exist before is what the run reports as observations added. + */ +export async function recordMemberObservations( + executor: DbOrTx, + memberId: string, + documentIds: readonly string[], + runId: string +): Promise { + let added = 0 + const now = new Date() + for (let offset = 0; offset < documentIds.length; offset += OBSERVATION_BATCH_SIZE) { + const batch = documentIds.slice(offset, offset + OBSERVATION_BATCH_SIZE) + const written = await executor + .insert(knowledgeDocumentObservation) + .values(batch.map((documentId) => ({ documentId, memberId, lastSeenAt: now, runId }))) + .onConflictDoUpdate({ + target: [knowledgeDocumentObservation.documentId, knowledgeDocumentObservation.memberId], + set: { lastSeenAt: now, runId }, + }) + .returning({ inserted: sql`(xmax = 0)` }) + added += written.filter((row) => row.inserted).length + } + return added +} + +/** + * Removes every observation of one member that this run did not re-assert. + * Only called after a full, complete, non-suspect listing: absence from any + * other kind of listing says nothing about access. + */ +export async function removeUnseenMemberObservations( + executor: DbOrTx, + memberId: string, + runId: string +): Promise { + const removed = await executor + .delete(knowledgeDocumentObservation) + .where( + and( + eq(knowledgeDocumentObservation.memberId, memberId), + ne(knowledgeDocumentObservation.runId, runId) + ) + ) + .returning({ documentId: knowledgeDocumentObservation.documentId }) + return removed.map((row) => row.documentId) +} + +/** + * Withdraws one member's observations of specific documents: what their + * change feed reported as deleted or no longer reachable. Returns the ids + * whose observation actually existed. + */ +export async function removeMemberObservationsForDocuments( + executor: DbOrTx, + memberId: string, + documentIds: readonly string[] +): Promise { + if (documentIds.length === 0) return [] + const removed: string[] = [] + for (let offset = 0; offset < documentIds.length; offset += OBSERVATION_BATCH_SIZE) { + const batch = documentIds.slice(offset, offset + OBSERVATION_BATCH_SIZE) + const rows = await executor + .delete(knowledgeDocumentObservation) + .where( + and( + eq(knowledgeDocumentObservation.memberId, memberId), + inArray(knowledgeDocumentObservation.documentId, batch) + ) + ) + .returning({ documentId: knowledgeDocumentObservation.documentId }) + for (const row of rows) removed.push(row.documentId) + } + return removed +} + +/** Every document the given members have observed, for rematerialisation after a membership change. */ +export async function listObservedDocumentIds( + executor: DbOrTx, + memberIds: readonly string[] +): Promise { + if (memberIds.length === 0) return [] + const rows = await executor + .selectDistinct({ documentId: knowledgeDocumentObservation.documentId }) + .from(knowledgeDocumentObservation) + .where(inArray(knowledgeDocumentObservation.memberId, memberIds)) + return rows.map((row) => row.documentId) +} + +/** + * Rewrites `document.acl` from the observation graph: the sorted subject + * tokens of every active observer, or nobody. Scoped to the connector so a + * document id that was detached or re-owned since it was collected is left + * alone. + */ +/** Documents rewritten per statement while a mode switch rewrites a connector's ACLs. */ +const ACCESS_REWRITE_BATCH_SIZE = 1000 + +/** + * Rewrites every document ACL of the connector to `target`, in bounded + * batches, until done or `deadlineAt` passes. Returns whether every row was + * rewritten. `beforeBatch` runs ahead of each statement, for a lease heartbeat. + */ +export async function rewriteConnectorAcls( + connectorId: string, + target: readonly string[], + options: { + deadlineAt?: number + beforeBatch?: () => Promise + /** + * The lease the caller holds on the connector, proved inside each batch's + * transaction: a heartbeat before the batch only says the lease was held + * then, and a run reclaimed mid-rewrite must not land an empty ACL over + * what its replacement has since materialised. + */ + lease?: SyncWriteLease + } = {} +): Promise { + const mismatch = + target.length === 0 + ? sql`cardinality(${document.acl}) > 0` + : sql`${document.acl} <> ${textArrayLiteral(target)}` + for (;;) { + await options.beforeBatch?.() + const rewritten = await db.transaction(async (tx) => { + if (options.lease) await assertSyncLeaseHeldInTx(tx, connectorId, options.lease) + return tx + .update(document) + .set({ acl: [...target] }) + .where( + eq( + document.id, + sql`ANY(ARRAY( + SELECT ${document.id} FROM ${document} + WHERE ${document.connectorId} = ${connectorId} AND ${mismatch} + LIMIT ${ACCESS_REWRITE_BATCH_SIZE} + ))` + ) + ) + .returning({ id: document.id }) + }) + if (rewritten.length < ACCESS_REWRITE_BATCH_SIZE) return true + if (options.deadlineAt !== undefined && Date.now() >= options.deadlineAt) return false + } +} + +export async function materializeDocumentAcls( + connectorId: string, + documentIds: Iterable, + executor: DbOrTx = db +): Promise { + const ids = [...new Set(documentIds)] + let updated = 0 + for (let offset = 0; offset < ids.length; offset += MATERIALIZE_BATCH_SIZE) { + const batch = ids.slice(offset, offset + MATERIALIZE_BATCH_SIZE) + const rows = await executor + .update(document) + .set({ acl: observedAcl() }) + .where( + and( + inArray(document.id, batch), + eq(document.connectorId, connectorId), + sql`${document.acl} IS DISTINCT FROM ${observedAcl()}` + ) + ) + .returning({ id: document.id }) + updated += rows.length + } + return updated +} + +export interface MemberDocumentLifecycleResult { + tombstoned: number + resurrected: number + purged: number +} + +/** + * The members-mode document lifecycle, applied idempotently every run: + * a document nobody observes (in any member state) is tombstoned, one that is + * observed again is resurrected, and one that has stayed unobserved past the + * purge window is hard deleted under the run's lease. Existence follows the + * observation graph; visibility follows the active observers through the ACL. + * + * A document whose content refresh failed this run is not resurrected: its + * stored content is known-stale, and surfacing it would show pre-tombstone + * content as current. It stays tombstoned for a later run to retry. + */ +export async function applyMemberDocumentLifecycle(input: { + connectorId: string + knowledgeBaseId: string + runId: string + lease: Pick + /** Runs the tombstone and resurrection writes only while the run still holds its lease. */ + withLease: (fn: (tx: DbOrTx) => Promise) => Promise + /** External ids whose refresh did not land this run; withheld from resurrection. */ + failedExternalIds: ReadonlySet + /** + * Whether absence of observers may hide or purge a document. False until at + * least one member has completed a listing: before that, nothing has been + * observed yet, so absence says nothing. + */ + allowRemoval: boolean +}): Promise { + const { connectorId, knowledgeBaseId, runId } = input + const now = new Date() + + const { tombstoned, resurrected } = await input.withLease(async (tx) => { + const tombstoned = !input.allowRemoval + ? [] + : await tx + .update(document) + .set({ deletedAt: now }) + .where( + and( + eq(document.connectorId, connectorId), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNull(document.deletedAt), + hasNoObservation() + ) + ) + .returning({ id: document.id }) + + const resurrectionCandidates = await tx + .select({ id: document.id, externalId: document.externalId }) + .from(document) + .where( + and( + eq(document.connectorId, connectorId), + isNull(document.archivedAt), + isNotNull(document.deletedAt), + hasObservation() + ) + ) + const resurrectIds = resurrectionCandidates + .filter((row) => !row.externalId || !input.failedExternalIds.has(row.externalId)) + .map((row) => row.id) + const resurrected = + resurrectIds.length === 0 + ? [] + : await tx + .update(document) + .set({ deletedAt: null }) + .where( + and( + inArray(document.id, resurrectIds), + eq(document.connectorId, connectorId), + isNull(document.archivedAt), + isNotNull(document.deletedAt) + ) + ) + .returning({ id: document.id }) + return { tombstoned, resurrected } + }) + + const purgeCutoff = new Date(now.getTime() - MEMBER_TOMBSTONE_PURGE_DAYS * 24 * 60 * 60 * 1000) + const purgeCandidates = input.allowRemoval + ? await db + .select({ id: document.id }) + .from(document) + .where( + and( + eq(document.connectorId, connectorId), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNotNull(document.deletedAt), + lt(document.deletedAt, purgeCutoff), + hasNoObservation() + ) + ) + .limit(MEMBER_PURGE_MAX_PER_RUN) + : [] + + const guard: ConnectorSyncDeletionGuard = { + connectorId, + knowledgeBaseId, + syncLockToken: runId, + lease: 'member', + } + let purged = 0 + const purgeIds = purgeCandidates.map((row) => row.id) + for (let offset = 0; offset < purgeIds.length; offset += PURGE_CHUNK_SIZE) { + await input.lease.beatIfDue() + try { + purged += await hardDeleteDocuments( + purgeIds.slice(offset, offset + PURGE_CHUNK_SIZE), + runId, + connectorId, + knowledgeBaseId, + guard + ) + } catch (error) { + /** The deletion guard refusing the lease is a reclaimed run, not a failed one. */ + if (error instanceof ConnectorSyncDeletionGuardError) { + throw new SyncLockLostException(connectorId) + } + throw error + } + } + + return { tombstoned: tombstoned.length, resurrected: resurrected.length, purged } +} + +export interface StaleMemberSweepResult { + members: number + observationsRemoved: number + documentsRematerialized: number + docsTombstoned: number +} + +/** How long a member's crawls may be silent before the sweep treats them as gone: `max(24 h, 2 × interval)`. */ +export function staleMemberWindowMs(syncIntervalMinutes: number): number { + return Math.max( + MEMBER_OBSERVATION_STALE_AFTER_HOURS * 60 * 60 * 1000, + 2 * syncIntervalMinutes * 60 * 1000 + ) +} + +/** The staleness a member is re-checked against once its row is locked; the same clock as the selection. */ +function memberStillStale(memberId: string, cutoff: Date) { + return and( + eq(knowledgeConnectorMember.id, memberId), + eq(knowledgeConnectorMember.status, 'active'), + or( + isNull(knowledgeConnectorMember.lastStartedAt), + lt(knowledgeConnectorMember.lastStartedAt, cutoff) + ), + or( + isNull(knowledgeConnectorMember.lastCompleteListingAt), + lt(knowledgeConnectorMember.lastCompleteListingAt, cutoff) + ) + ) +} + +/** + * Removes the observations of members whose crawls have stopped, so the + * documents only they observed go dark instead of staying readable forever. + * + * Fail-closed but schedule-relative: an active member is swept only when the + * connector itself completed a run inside `max(24 h, 2 × interval)` while both + * the member's last start and last complete listing are older than that, so + * queue lag in a large group, a deferred connector, or one on its failure + * ladder never trips it. A + * suspended member is not swept at all: suspension already drops their token + * from every ACL, and their observations are kept so a re-auth restores access + * without a re-crawl until membership reconciliation purges the row after + * `MEMBER_SUSPENDED_PURGE_DAYS`. The member row survives; the next run that + * lists for them rebuilds their observations. Purging is left to a run holding + * the lease. + * + * Each member is swept in one transaction that first shares the connector row + * — which a member run holds `FOR UPDATE` while it writes and a mode switch + * updates when it flips — and then locks the member row, which `claimNextMember` + * skips while locked. Both are re-checked under those locks, so a run that + * claimed the member after the selection, or a switch that left members mode, + * makes the sweep skip rather than delete observations a run just wrote or + * rewrite ACLs the switch just set. + */ +export async function sweepStaleMemberObservations(now: Date): Promise { + const staleWindow = sql`GREATEST( + ${MEMBER_OBSERVATION_STALE_AFTER_HOURS} * INTERVAL '1 hour', + 2 * ${knowledgeConnector.syncIntervalMinutes} * INTERVAL '1 minute' + )` + const cutoff = sql`${sql.param(now, knowledgeConnectorMember.lastStartedAt)} - ${staleWindow}` + const staleMembers = await db + .select({ + id: knowledgeConnectorMember.id, + connectorId: knowledgeConnectorMember.connectorId, + syncIntervalMinutes: knowledgeConnector.syncIntervalMinutes, + }) + .from(knowledgeConnectorMember) + .innerJoin(knowledgeConnector, eq(knowledgeConnector.id, knowledgeConnectorMember.connectorId)) + .where( + and( + eq(knowledgeConnector.accessMode, 'members'), + /** Only a connector that is meant to be crawling can have stopped crawling. */ + inArray(knowledgeConnector.status, MEMBER_LOCKABLE_CONNECTOR_STATUSES), + gt(knowledgeConnector.syncIntervalMinutes, 0), + connectorIsLive(), + /** + * Only a connector that is still completing runs can have left a member + * behind; one that is deferred, disabled, or backing off keeps every + * observation until it runs again. + */ + ne(knowledgeConnector.memberSyncStatus, 'disabled'), + gt(knowledgeConnector.lastMemberSyncAt, cutoff), + exists( + db + .select({ one: sql`1` }) + .from(knowledgeDocumentObservation) + .where(eq(knowledgeDocumentObservation.memberId, knowledgeConnectorMember.id)) + ), + eq(knowledgeConnectorMember.status, 'active'), + lt(knowledgeConnectorMember.createdAt, cutoff), + or( + isNull(knowledgeConnectorMember.lastStartedAt), + lt(knowledgeConnectorMember.lastStartedAt, cutoff) + ), + or( + isNull(knowledgeConnectorMember.lastCompleteListingAt), + lt(knowledgeConnectorMember.lastCompleteListingAt, cutoff) + ) + ) + ) + .limit(STALE_MEMBER_SWEEP_LIMIT) + + const result: StaleMemberSweepResult = { + members: 0, + observationsRemoved: 0, + documentsRematerialized: 0, + docsTombstoned: 0, + } + for (const member of staleMembers) { + const memberCutoff = new Date(now.getTime() - staleMemberWindowMs(member.syncIntervalMinutes)) + const swept = await db.transaction(async (tx) => { + const [connector] = await tx + .select({ id: knowledgeConnector.id }) + .from(knowledgeConnector) + .where( + and( + eq(knowledgeConnector.id, member.connectorId), + eq(knowledgeConnector.accessMode, 'members'), + inArray(knowledgeConnector.status, MEMBER_LOCKABLE_CONNECTOR_STATUSES), + ne(knowledgeConnector.memberSyncStatus, 'disabled'), + connectorIsLive() + ) + ) + .for('share') + if (!connector) return null + const [stale] = await tx + .select({ id: knowledgeConnectorMember.id }) + .from(knowledgeConnectorMember) + .where(memberStillStale(member.id, memberCutoff)) + .for('update') + if (!stale) return null + + const removed = await tx + .delete(knowledgeDocumentObservation) + .where(eq(knowledgeDocumentObservation.memberId, member.id)) + .returning({ documentId: knowledgeDocumentObservation.documentId }) + const documentIds = removed.map((row) => row.documentId) + const rematerialized = await materializeDocumentAcls(member.connectorId, documentIds, tx) + const tombstoned = + documentIds.length === 0 + ? [] + : await tx + .update(document) + .set({ deletedAt: now }) + .where( + and( + inArray(document.id, documentIds), + eq(document.connectorId, member.connectorId), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNull(document.deletedAt), + hasNoObservation() + ) + ) + .returning({ id: document.id }) + return { + observationsRemoved: documentIds.length, + documentsRematerialized: rematerialized, + docsTombstoned: tombstoned.length, + } + }) + if (!swept) continue + result.members += 1 + result.observationsRemoved += swept.observationsRemoved + result.documentsRematerialized += swept.documentsRematerialized + result.docsTombstoned += swept.docsTombstoned + } + return result +} diff --git a/apps/sim/lib/knowledge/connectors/member-provisioning.test.ts b/apps/sim/lib/knowledge/connectors/member-provisioning.test.ts new file mode 100644 index 00000000000..a7aa1891de9 --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/member-provisioning.test.ts @@ -0,0 +1,90 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('@/lib/credential-groups/enrollments', () => ({ + createCredentialGroupInvitationLink: vi.fn(), + inviteCredentialGroupEnrollment: vi.fn(), +})) +vi.mock('@/lib/knowledge/access/availability', () => ({ + isKnowledgeMemberAccessAvailable: vi.fn(), +})) +vi.mock('@/lib/knowledge/connectors/member-queue', () => ({ dispatchMemberSync: vi.fn() })) +vi.mock('@/lib/billing/core/billing-attribution', () => ({ + resolveSystemBillingAttribution: vi.fn(), +})) +vi.mock('@/lib/credential-groups/service', () => ({ + createCredentialGroup: vi.fn(), + listCredentialGroups: vi.fn(), +})) +vi.mock('@/lib/workspaces/permissions/utils', () => ({ getUsersWithPermissions: vi.fn() })) + +import { + chooseSharedMembersBinding, + deriveViewerConnectorMembership, + pickProvisionedGroupName, +} from '@/lib/knowledge/connectors/member-provisioning' + +describe('pickProvisionedGroupName', () => { + it('names the group after the connector and steps past taken names', () => { + expect(pickProvisionedGroupName('Google Drive', [])).toBe('Google Drive') + expect(pickProvisionedGroupName('Google Drive', ['google drive'])).toBe('Google Drive 2') + expect(pickProvisionedGroupName('Google Drive', ['Google Drive', 'Google Drive 2'])).toBe( + 'Google Drive 3' + ) + }) + + it('gives up with a pointer to Settings once every candidate is taken', () => { + const taken = [ + 'Google Drive', + 'Google Drive 2', + 'Google Drive 3', + 'Google Drive 4', + 'Google Drive 5', + ] + expect(() => pickProvisionedGroupName('Google Drive', taken)).toThrow('Settings') + }) +}) + +describe('chooseSharedMembersBinding', () => { + const a = { credentialGroupId: 'g1', credentialGroupOptionId: 'o1' } + const b = { credentialGroupId: 'g2', credentialGroupOptionId: 'o2' } + + it('reuses the option other members-mode connectors sync through', () => { + expect(chooseSharedMembersBinding([a, b], new Set(['o2']))).toBe(b) + }) + + it('creates a new group rather than repurpose one nobody syncs through', () => { + expect(chooseSharedMembersBinding([a, b], new Set())).toBeUndefined() + expect(chooseSharedMembersBinding([], new Set())).toBeUndefined() + }) + + it('leaves two shared options for the caller to choose between', () => { + expect(chooseSharedMembersBinding([a, b], new Set(['o1', 'o2']))).toBeNull() + }) +}) + +describe('deriveViewerConnectorMembership', () => { + it.each([ + [true, 'active', 'completed', 'connected'], + [true, 'active', 'in_progress', 'connected'], + [true, 'needs_reauth', 'completed', 'needs_reauth'], + [true, null, 'invited', 'invited'], + [true, null, 'delivery_failed', 'invited'], + [true, null, 'in_progress', 'invited'], + [true, null, 'completed', 'invited'], + [true, 'revoked', 'completed', 'invited'], + [true, 'active', 'revoked', 'revoked'], + [true, null, 'revoked', 'revoked'], + [true, null, null, 'not_enrolled'], + [false, 'active', 'completed', 'unverified_email'], + ] as const)( + 'verified %s + credential %s + enrollment %s → %s', + (emailVerified, managedOauthStatus, enrollmentStatus, expected) => { + expect( + deriveViewerConnectorMembership({ emailVerified, managedOauthStatus, enrollmentStatus }) + ).toBe(expected) + } + ) +}) diff --git a/apps/sim/lib/knowledge/connectors/member-provisioning.ts b/apps/sim/lib/knowledge/connectors/member-provisioning.ts new file mode 100644 index 00000000000..6d5e93a9d20 --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/member-provisioning.ts @@ -0,0 +1,394 @@ +import { db } from '@sim/db' +import { + credential, + credentialGroupEnrollment, + knowledgeBase, + knowledgeConnector, + user, +} from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { normalizeEmail } from '@sim/utils/string' +import { and, eq, inArray, isNull } from 'drizzle-orm' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { + CredentialGroupEnrollmentError, + createCredentialGroupInvitationLink, + inviteCredentialGroupEnrollment, +} from '@/lib/credential-groups/enrollments' +import { + getCredentialGroupProviderId, + getCredentialGroupStandardOAuthProviderFromProviderId, + isCredentialGroupProvider, +} from '@/lib/credential-groups/providers' +import { createCredentialGroup, listCredentialGroups } from '@/lib/credential-groups/service' +import { isKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' +import { getUsersWithPermissions } from '@/lib/workspaces/permissions/utils' +import type { ConnectorMeta } from '@/connectors/types' + +const logger = createLogger('KnowledgeConnectorMemberProvisioning') + +/** Invitations sent between two lease heartbeats of a member run. */ +const INVITATION_BATCH_SIZE = 25 +/** Names tried for the group a connector provisions, in order. */ +const PROVISIONED_GROUP_NAME_ATTEMPTS = 5 + +export interface ProvisionedMembersBinding { + credentialGroupId: string + credentialGroupOptionId: string +} + +/** + * The name of the group a connector provisions: the connector's own name, + * which is what the invitation email and the enrollment page show, suffixed + * only when the workspace already uses it. + */ +export function pickProvisionedGroupName( + connectorName: string, + takenNames: readonly string[] +): string { + const taken = new Set(takenNames.map((name) => name.trim().toLocaleLowerCase())) + for (let attempt = 1; attempt <= PROVISIONED_GROUP_NAME_ATTEMPTS; attempt++) { + const candidate = attempt === 1 ? connectorName : `${connectorName} ${attempt}` + if (!taken.has(candidate.toLocaleLowerCase())) return candidate + } + throw new OrchestrationError( + 'conflict', + `Every name from "${connectorName}" to "${connectorName} ${PROVISIONED_GROUP_NAME_ATTEMPTS}" is taken; pick a Credential Group in Settings` + ) +} + +/** + * Among the workspace's active options collecting the connector's accounts, + * the one other members-mode connectors already sync through, so one + * connection serves every connector of a provider. A group nobody syncs + * through is never reused: it was curated for something else, and joining + * it would invite the whole workspace to it. Returns `undefined` when a new + * group is needed and `null` when two shared options make the choice + * ambiguous. + */ +export function chooseSharedMembersBinding( + candidates: readonly ProvisionedMembersBinding[], + optionIdsServingMemberConnectors: ReadonlySet +): ProvisionedMembersBinding | null | undefined { + const shared = candidates.filter((candidate) => + optionIdsServingMemberConnectors.has(candidate.credentialGroupOptionId) + ) + if (shared.length === 1) return shared[0] + return shared.length > 1 ? null : undefined +} + +async function listOptionIdsServingMemberConnectors( + workspaceId: string, + optionIds: readonly string[] +): Promise> { + if (optionIds.length === 0) return new Set() + const rows = await db + .select({ optionId: knowledgeConnector.credentialGroupOptionId }) + .from(knowledgeConnector) + .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) + .where( + and( + eq(knowledgeBase.workspaceId, workspaceId), + eq(knowledgeConnector.accessMode, 'members'), + inArray(knowledgeConnector.credentialGroupOptionId, [...optionIds]), + isNull(knowledgeConnector.deletedAt) + ) + ) + return new Set(rows.flatMap((row) => (row.optionId ? [row.optionId] : []))) +} + +/** + * The Credential Group option a members-mode connector crawls through when + * the caller named none: the option this provider's other members-mode + * connectors share, or a group created for the purpose. + */ +export async function provisionKnowledgeConnectorMembersBinding(input: { + workspaceId: string + connectorMeta: Pick + userId: string +}): Promise { + const { connectorMeta } = input + if (connectorMeta.auth.mode !== 'oauth') { + throw new OrchestrationError('validation', 'Only an OAuth connector can sync per member') + } + const providerId = connectorMeta.auth.provider + let provider: ReturnType + try { + provider = getCredentialGroupStandardOAuthProviderFromProviderId(providerId) + } catch { + throw new OrchestrationError( + 'validation', + `${connectorMeta.name} accounts cannot be collected through a Credential Group yet` + ) + } + + const groups = await listCredentialGroups(input.workspaceId) + const candidates: ProvisionedMembersBinding[] = [] + for (const group of groups) { + if (group.status !== 'active') continue + for (const option of group.options) { + if (option.status !== 'active') continue + if (!isCredentialGroupProvider(option.provider)) continue + if (getCredentialGroupProviderId(option.provider) !== providerId) continue + candidates.push({ credentialGroupId: group.id, credentialGroupOptionId: option.id }) + } + } + const shared = chooseSharedMembersBinding( + candidates, + await listOptionIdsServingMemberConnectors( + input.workspaceId, + candidates.map((candidate) => candidate.credentialGroupOptionId) + ) + ) + if (shared) return shared + if (shared === null) { + throw new OrchestrationError( + 'validation', + `Several Credential Groups collect ${connectorMeta.name} accounts for other connectors; choose which one this connector syncs through` + ) + } + + const name = pickProvisionedGroupName( + connectorMeta.name, + groups.map((group) => group.name) + ) + const group = await createCredentialGroup(input.workspaceId, input.userId, { + name, + options: [{ provider, label: connectorMeta.name, required: true }], + }) + const option = group.options[0] + if (!option) throw new Error('Provisioned Credential Group has no option') + logger.info('Provisioned a Credential Group for a members-mode connector', { + workspaceId: input.workspaceId, + credentialGroupId: group.id, + provider, + }) + return { credentialGroupId: group.id, credentialGroupOptionId: option.id } +} + +export interface InviteWorkspaceMembersResult { + invited: number + failed: number +} + +/** + * Invites every workspace member who has no enrollment in the group yet, so + * joining the workspace is all a person has to do before connecting their + * account. An enrollment an admin revoked is left alone — the invitation is + * issued with `reject`, so a revocation that lands after the enrollments were + * read is refused inside the issuing transaction rather than reactivated. + * Runs inside a member run: `beforeBatch` beats the run's lease between + * batches, and failures are logged per person rather than aborting the run. + */ +export async function inviteWorkspaceMembersToCredentialGroup(input: { + workspaceId: string + credentialGroupId: string + beforeBatch: () => Promise +}): Promise { + const [members, enrolled] = await Promise.all([ + getUsersWithPermissions(input.workspaceId), + db + .select({ email: credentialGroupEnrollment.email }) + .from(credentialGroupEnrollment) + .where(eq(credentialGroupEnrollment.credentialGroupId, input.credentialGroupId)), + ]) + const enrolledEmails = new Set(enrolled.map((row) => normalizeEmail(row.email))) + const pending = [...new Set(members.map((member) => normalizeEmail(member.email)))].filter( + (email) => email && !enrolledEmails.has(email) + ) + + const result: InviteWorkspaceMembersResult = { invited: 0, failed: 0 } + for (let offset = 0; offset < pending.length; offset += INVITATION_BATCH_SIZE) { + await input.beforeBatch() + for (const email of pending.slice(offset, offset + INVITATION_BATCH_SIZE)) { + try { + await inviteCredentialGroupEnrollment( + input.workspaceId, + input.credentialGroupId, + undefined, + undefined, + email, + 'reject' + ) + result.invited += 1 + } catch (error) { + result.failed += 1 + logger.warn('Failed to invite a workspace member to a connector credential group', { + workspaceId: input.workspaceId, + credentialGroupId: input.credentialGroupId, + error: getErrorMessage(error), + }) + } + } + } + return result +} + +/** + * Where a viewer stands with a members-mode connector, from their account + * and their enrollment in the connector's group. + */ +export type ViewerConnectorMembership = + | 'connected' + | 'needs_reauth' + | 'invited' + | 'not_enrolled' + | 'revoked' + | 'unverified_email' + +export function deriveViewerConnectorMembership(input: { + emailVerified: boolean + enrollmentStatus: string | null + managedOauthStatus: string | null +}): ViewerConnectorMembership { + if (!input.emailVerified) return 'unverified_email' + if (input.enrollmentStatus === 'revoked') return 'revoked' + if (input.managedOauthStatus === 'active') return 'connected' + if (input.managedOauthStatus === 'needs_reauth') return 'needs_reauth' + if (input.enrollmentStatus) return 'invited' + return 'not_enrolled' +} + +/** + * The viewer's membership in each members-mode connector, keyed by connector + * id. Connectors that sync as the workspace are absent, and so is everything + * where the feature is off: there is nothing the viewer could connect to. + */ +export async function resolveViewerConnectorMemberships(input: { + userId: string + workspaceId: string + connectors: ReadonlyArray<{ + id: string + accessMode: string + credentialGroupId: string | null + credentialGroupOptionId: string | null + }> +}): Promise> { + const result = new Map() + const memberConnectors = input.connectors.filter( + (connector) => + connector.accessMode === 'members' && + connector.credentialGroupId && + connector.credentialGroupOptionId + ) + if (memberConnectors.length === 0) return result + if (!(await isKnowledgeMemberAccessAvailable({ workspaceId: input.workspaceId }))) return result + + const [viewer] = await db + .select({ email: user.email, emailVerified: user.emailVerified }) + .from(user) + .where(eq(user.id, input.userId)) + .limit(1) + if (!viewer) return result + const email = normalizeEmail(viewer.email) + const groupIds = [...new Set(memberConnectors.map((connector) => connector.credentialGroupId!))] + const rows = await db + .select({ + credentialGroupId: credentialGroupEnrollment.credentialGroupId, + enrollmentStatus: credentialGroupEnrollment.status, + credentialGroupOptionId: credential.credentialGroupOptionId, + managedOauthStatus: credential.managedOauthStatus, + }) + .from(credentialGroupEnrollment) + .leftJoin( + credential, + and( + eq(credential.credentialGroupEnrollmentId, credentialGroupEnrollment.id), + eq(credential.workspaceId, input.workspaceId), + eq(credential.type, 'managed_oauth') + ) + ) + .where( + and( + inArray(credentialGroupEnrollment.credentialGroupId, groupIds), + eq(credentialGroupEnrollment.email, email) + ) + ) + + for (const connector of memberConnectors) { + const enrollment = rows.find((row) => row.credentialGroupId === connector.credentialGroupId) + const forOption = rows.find( + (row) => + row.credentialGroupId === connector.credentialGroupId && + row.credentialGroupOptionId === connector.credentialGroupOptionId + ) + result.set( + connector.id, + deriveViewerConnectorMembership({ + emailVerified: viewer.emailVerified, + enrollmentStatus: enrollment?.enrollmentStatus ?? null, + managedOauthStatus: forOption?.managedOauthStatus ?? null, + }) + ) + } + return result +} + +/** + * A fresh enrollment link for the viewer into the connector's group, minted + * on demand so a workspace member never has to find the invitation email. + * Issued without an inviter — the person is inviting themselves — and refused + * for an enrollment an admin revoked or an account whose email is unverified, + * which could connect but would never be granted a token. The revocation is + * decided inside the issuing transaction (`reject`), so an admin who revokes + * between the read here and the issue is never overridden by a link. + */ +export async function createViewerConnectorEnrollmentLink(input: { + userId: string + workspaceId: string + credentialGroupId: string +}): Promise { + const [viewer] = await db + .select({ email: user.email, emailVerified: user.emailVerified }) + .from(user) + .where(eq(user.id, input.userId)) + .limit(1) + if (!viewer) throw new OrchestrationError('not_found', 'User not found') + if (!viewer.emailVerified) { + throw new OrchestrationError( + 'validation', + 'Verify your email address before connecting an account' + ) + } + const email = normalizeEmail(viewer.email) + const revoked = new OrchestrationError( + 'forbidden', + 'A workspace admin removed your access to this connector' + ) + if (await isEnrollmentRevoked(input.credentialGroupId, email)) throw revoked + try { + const { invitationLink } = await createCredentialGroupInvitationLink( + input.workspaceId, + input.credentialGroupId, + undefined, + email, + 'reject' + ) + return invitationLink + } catch (error) { + /** The issue refused a revocation that landed after the read above; report it as such. */ + if ( + error instanceof CredentialGroupEnrollmentError && + error.status === 409 && + (await isEnrollmentRevoked(input.credentialGroupId, email)) + ) { + throw revoked + } + throw error + } +} + +async function isEnrollmentRevoked(credentialGroupId: string, email: string): Promise { + const [enrollment] = await db + .select({ status: credentialGroupEnrollment.status }) + .from(credentialGroupEnrollment) + .where( + and( + eq(credentialGroupEnrollment.credentialGroupId, credentialGroupId), + eq(credentialGroupEnrollment.email, email) + ) + ) + .limit(1) + return enrollment?.status === 'revoked' +} diff --git a/apps/sim/lib/knowledge/connectors/member-queue.test.ts b/apps/sim/lib/knowledge/connectors/member-queue.test.ts new file mode 100644 index 00000000000..68567ff7ee3 --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/member-queue.test.ts @@ -0,0 +1,319 @@ +/** + * @vitest-environment node + */ +import { dbChainMockFns, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockExecuteMemberSync, + mockIsTriggerAvailable, + mockTrigger, + mockResolveRegion, + mockResolveSystemBilling, +} = vi.hoisted(() => ({ + mockExecuteMemberSync: vi.fn(), + mockIsTriggerAvailable: vi.fn(), + mockTrigger: vi.fn(), + mockResolveRegion: vi.fn(), + mockResolveSystemBilling: vi.fn(), +})) + +vi.mock('@/lib/billing/core/billing-attribution', () => ({ + assertBillingAttributionSnapshot: (value: unknown) => value, + resolveSystemBillingAttribution: mockResolveSystemBilling, +})) +vi.mock('@/lib/knowledge/connectors/member-sync-engine', () => ({ + executeMemberSync: mockExecuteMemberSync, +})) +vi.mock('@/lib/knowledge/documents/service', () => ({ + isTriggerAvailable: mockIsTriggerAvailable, +})) +vi.mock('@trigger.dev/sdk', () => ({ + tasks: { trigger: mockTrigger }, + idempotencyKeys: { create: vi.fn(async (key: string) => key) }, +})) +vi.mock('@/lib/core/async-jobs/region', () => ({ resolveTriggerRegion: mockResolveRegion })) + +import { eq, inArray } from 'drizzle-orm' +import { + assertMemberSyncPayload, + dispatchMemberSync, + dispatchMemberSyncsForCredentialOption, + MEMBER_SYNC_TASK_ID, +} from '@/lib/knowledge/connectors/member-queue' + +const BILLING = { + actorUserId: 'user-1', + workspaceId: 'ws-1', + organizationId: null, + billedAccountUserId: 'owner-1', + billingEntity: { type: 'user' as const, id: 'owner-1' }, + billingPeriod: { start: '2026-09-01T00:00:00.000Z', end: '2026-10-01T00:00:00.000Z' }, + payerSubscription: null, +} + +const CONNECTOR_ROW = { + knowledgeBaseId: 'kb-1', + accessMode: 'members', + status: 'active', + memberSyncStatus: 'idle', + nextMemberSyncAt: null, + archivedAt: null, + deletedAt: null, + workspaceId: 'ws-1', + kbDeletedAt: null, +} + +describe('member sync queue', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mockIsTriggerAvailable.mockReturnValue(true) + mockResolveRegion.mockResolvedValue('us') + mockExecuteMemberSync.mockResolvedValue({}) + }) + + describe('assertMemberSyncPayload', () => { + it('restores a well-formed payload', () => { + expect( + assertMemberSyncPayload({ + connectorId: 'c-1', + requestId: 'r-1', + billingAttribution: BILLING, + dispatchToken: 't-1', + }) + ).toEqual({ + connectorId: 'c-1', + requestId: 'r-1', + billingAttribution: BILLING, + dispatchToken: 't-1', + }) + }) + + it.each([ + ['no connector', { requestId: 'r-1', billingAttribution: BILLING }], + ['no request id', { connectorId: 'c-1', billingAttribution: BILLING }], + ['no billing attribution', { connectorId: 'c-1', requestId: 'r-1' }], + [ + 'a blank token', + { connectorId: 'c-1', requestId: 'r-1', billingAttribution: BILLING, dispatchToken: ' ' }, + ], + ])('rejects a payload with %s', (_name, payload) => { + expect(() => assertMemberSyncPayload(payload)).toThrow() + }) + }) + + describe('dispatchMemberSync', () => { + it('takes the queue entry and hands the run to the queue with its token', async () => { + queueTableRows(schemaMock.knowledgeConnector, [CONNECTOR_ROW]) + dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'c-1' }]) + + await expect( + dispatchMemberSync('c-1', { billingAttribution: BILLING, requestId: 'r-1' }) + ).resolves.toEqual({ queued: true }) + + expect(mockTrigger).toHaveBeenCalledWith( + MEMBER_SYNC_TASK_ID, + expect.objectContaining({ + connectorId: 'c-1', + requestId: 'r-1', + dispatchToken: expect.any(String), + }), + expect.objectContaining({ region: 'us' }) + ) + expect(dbChainMockFns.set).toHaveBeenCalledWith( + expect.objectContaining({ + memberSyncStatus: 'pending', + memberSyncLockToken: expect.any(String), + }) + ) + }) + + it('runs in-process with the token when the queue is unavailable', async () => { + mockIsTriggerAvailable.mockReturnValue(false) + queueTableRows(schemaMock.knowledgeConnector, [CONNECTOR_ROW]) + dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'c-1' }]) + + await expect( + dispatchMemberSync('c-1', { billingAttribution: BILLING, requestId: 'r-1' }) + ).resolves.toEqual({ queued: true }) + + expect(mockTrigger).not.toHaveBeenCalled() + expect(mockExecuteMemberSync).toHaveBeenCalledWith( + 'c-1', + expect.objectContaining({ billingAttribution: BILLING, dispatchToken: expect.any(String) }) + ) + }) + + it('releases its own queue entry when the hand-off throws', async () => { + queueTableRows(schemaMock.knowledgeConnector, [CONNECTOR_ROW]) + dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'c-1' }]) + mockTrigger.mockRejectedValueOnce(new Error('queue down')) + + await expect( + dispatchMemberSync('c-1', { billingAttribution: BILLING, requestId: 'r-1' }) + ).rejects.toThrow('queue down') + + expect(dbChainMockFns.set).toHaveBeenLastCalledWith( + expect.objectContaining({ + memberSyncStatus: 'error', + memberSyncLockToken: null, + memberSyncLockLeaseAt: null, + }) + ) + }) + + it.each([ + ['a workspace-mode connector', { accessMode: 'workspace' }, 'does not sync per member'], + ['an archived connector', { archivedAt: new Date() }, 'archived or deleted'], + [ + 'a running connector on the automatic path', + { memberSyncStatus: 'running' }, + 'is running and is not run automatically', + ], + [ + 'a changed schedule on the automatic path', + { nextMemberSyncAt: new Date('2026-09-01T00:00:00Z') }, + 'schedule changed', + ], + ])('refuses %s without touching the queue', async (_name, overrides, reason) => { + queueTableRows(schemaMock.knowledgeConnector, [{ ...CONNECTOR_ROW, ...overrides }]) + + const result = await dispatchMemberSync('c-1', { + billingAttribution: BILLING, + requestId: 'r-1', + requireRunnable: true, + expectedNextMemberSyncAt: new Date('2026-09-01T06:00:00Z'), + }) + + expect(result.queued).toBe(false) + expect(result.reason).toContain(reason) + expect(mockTrigger).not.toHaveBeenCalled() + expect(dbChainMockFns.update).not.toHaveBeenCalled() + }) + + it('explains a queue entry it could not take', async () => { + queueTableRows(schemaMock.knowledgeConnector, [CONNECTOR_ROW]) + dbChainMockFns.returning.mockResolvedValueOnce([]) + queueTableRows(schemaMock.knowledgeConnector, [ + { + accessMode: 'members', + status: 'active', + memberSyncStatus: 'idle', + syncLockToken: 'content-run', + archivedAt: null, + deletedAt: null, + }, + ]) + + await expect( + dispatchMemberSync('c-1', { billingAttribution: BILLING, requestId: 'r-1' }) + ).resolves.toEqual({ + queued: false, + reason: 'A workspace sync is still running for this connector', + }) + expect(mockTrigger).not.toHaveBeenCalled() + }) + + it('refuses billing attribution for another workspace', async () => { + queueTableRows(schemaMock.knowledgeConnector, [{ ...CONNECTOR_ROW, workspaceId: 'ws-2' }]) + + await expect( + dispatchMemberSync('c-1', { billingAttribution: BILLING, requestId: 'r-1' }) + ).rejects.toThrow('does not match connector workspace ws-2') + }) + + /** + * The guards above the CAS read the row once; a pause or a schedule change + * that lands after that read is only visible to the CAS itself. + */ + it('decides the connector status and the schedule inside the queue CAS', async () => { + const expected = new Date('2026-09-01T06:00:00Z') + queueTableRows(schemaMock.knowledgeConnector, [ + { ...CONNECTOR_ROW, nextMemberSyncAt: expected }, + ]) + dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'c-1' }]) + + await expect( + dispatchMemberSync('c-1', { + billingAttribution: BILLING, + requestId: 'r-1', + requireRunnable: true, + expectedNextMemberSyncAt: expected, + }) + ).resolves.toEqual({ queued: true }) + + expect(vi.mocked(inArray)).toHaveBeenCalledWith(schemaMock.knowledgeConnector.status, [ + 'active', + 'error', + ]) + expect(vi.mocked(eq)).toHaveBeenCalledWith( + schemaMock.knowledgeConnector.nextMemberSyncAt, + expected + ) + }) + + it.each([ + [ + 'a connector paused after the read', + { status: 'paused', nextMemberSyncAt: new Date('2026-09-01T06:00:00Z') }, + 'Connector is paused and is not synced', + ], + [ + 'a schedule that moved after the read', + { status: 'active', nextMemberSyncAt: new Date('2026-09-01T07:00:00Z') }, + 'The member sync schedule changed after this run was scheduled', + ], + ])('explains a queue entry refused for %s', async (_name, current, reason) => { + const expected = new Date('2026-09-01T06:00:00Z') + queueTableRows(schemaMock.knowledgeConnector, [ + { ...CONNECTOR_ROW, nextMemberSyncAt: expected }, + ]) + dbChainMockFns.returning.mockResolvedValueOnce([]) + queueTableRows(schemaMock.knowledgeConnector, [ + { + accessMode: 'members', + memberSyncStatus: 'idle', + syncLockToken: null, + archivedAt: null, + deletedAt: null, + ...current, + }, + ]) + + await expect( + dispatchMemberSync('c-1', { + billingAttribution: BILLING, + requestId: 'r-1', + requireRunnable: true, + expectedNextMemberSyncAt: expected, + }) + ).resolves.toEqual({ queued: false, reason }) + expect(mockTrigger).not.toHaveBeenCalled() + }) + }) + + describe('dispatchMemberSyncsForCredentialOption', () => { + it('keeps dispatching the remaining connectors when one hand-off throws', async () => { + mockResolveSystemBilling.mockResolvedValue(BILLING) + queueTableRows(schemaMock.knowledgeConnector, [{ id: 'c-1' }, { id: 'c-2' }]) + queueTableRows(schemaMock.knowledgeConnector, [{ ...CONNECTOR_ROW, workspaceId: 'ws-2' }]) + queueTableRows(schemaMock.knowledgeConnector, [CONNECTOR_ROW]) + dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'c-2' }]) + + await expect( + dispatchMemberSyncsForCredentialOption({ + workspaceId: 'ws-1', + credentialGroupOptionId: 'opt-1', + }) + ).resolves.toBeUndefined() + + expect(mockTrigger).toHaveBeenCalledOnce() + expect(mockTrigger).toHaveBeenCalledWith( + MEMBER_SYNC_TASK_ID, + expect.objectContaining({ connectorId: 'c-2' }), + expect.any(Object) + ) + }) + }) +}) diff --git a/apps/sim/lib/knowledge/connectors/member-queue.ts b/apps/sim/lib/knowledge/connectors/member-queue.ts new file mode 100644 index 00000000000..ea04f785163 --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/member-queue.ts @@ -0,0 +1,395 @@ +import { db } from '@sim/db' +import { knowledgeBase, knowledgeConnector } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { isRecordLike } from '@sim/utils/object' +import { idempotencyKeys, tasks } from '@trigger.dev/sdk' +import { and, eq, inArray, isNull } from 'drizzle-orm' +import { + assertBillingAttributionSnapshot, + type BillingAttributionSnapshot, + resolveSystemBillingAttribution, +} from '@/lib/billing/core/billing-attribution' +import { resolveTriggerRegion } from '@/lib/core/async-jobs/region' +import { executeMemberSync } from '@/lib/knowledge/connectors/member-sync-engine' +import { + SYNC_DISPATCH_FAILED_ERROR, + type SyncDispatchResult, +} from '@/lib/knowledge/connectors/queue' +import { + connectorIsLive, + MEMBER_LOCKABLE_CONNECTOR_STATUSES, +} from '@/lib/knowledge/connectors/sync-lock' +import { isTriggerAvailable } from '@/lib/knowledge/documents/service' + +const logger = createLogger('ConnectorMemberSyncQueue') + +export const MEMBER_SYNC_TASK_ID = 'knowledge-connector-member-sync' + +/** + * Member-sync states a run may be queued from. `pending` is deliberately + * absent here and present in the engine's lock acquisition: a queue entry is + * taken once, and the run that consumes it proves it by token. + */ +export const QUEUEABLE_MEMBER_SYNC_STATUSES = ['idle', 'error'] as const + +export interface MemberSyncPayload { + connectorId: string + requestId: string + billingAttribution: BillingAttributionSnapshot + /** The queue entry this task is allowed to consume; see `ConnectorSyncPayload.dispatchToken`. */ + dispatchToken?: string +} + +export interface DispatchMemberSyncOptions { + billingAttribution: BillingAttributionSnapshot + /** The scheduled instant this dispatch was made for; a changed schedule makes it stale. */ + expectedNextMemberSyncAt?: Date + /** Skip automatic work unless the connector is idle or recovering from an error. */ + requireRunnable?: boolean + requestId?: string +} + +function isNonEmptyString(value: unknown): value is string { + return typeof value === 'string' && value.trim().length > 0 +} + +/** Restores and validates member-sync work crossing the asynchronous boundary. */ +export function assertMemberSyncPayload(value: unknown): MemberSyncPayload { + if (!isRecordLike(value)) { + throw new Error('Member sync payload must be an object') + } + if (!isNonEmptyString(value.connectorId) || !isNonEmptyString(value.requestId)) { + throw new Error('Member sync payload requires connectorId and requestId') + } + if (value.dispatchToken !== undefined && !isNonEmptyString(value.dispatchToken)) { + throw new Error('Member sync payload dispatchToken must be a string when provided') + } + if (value.billingAttribution === undefined) { + throw new Error('Member sync payload requires billing attribution') + } + return { + connectorId: value.connectorId, + requestId: value.requestId, + billingAttribution: assertBillingAttributionSnapshot(value.billingAttribution), + dispatchToken: value.dispatchToken as string | undefined, + } +} + +/** + * Takes the member-sync queue entry, mirroring `markSyncPending` over the + * member lease columns. Refuses while the content engine holds its lock, so + * the two engines can never be queued against one connector at once, and + * refuses a connector that is no longer runnable or whose schedule moved + * since the dispatch read it: the guards above this CAS cannot see a pause + * or a schedule change that lands after they ran, so the CAS is where those + * are decided. + */ +async function markMemberSyncPending( + connectorId: string, + expectedNextMemberSyncAt: Date | undefined +): Promise { + const dispatchToken = generateId() + const now = new Date() + const taken = await db + .update(knowledgeConnector) + .set({ + memberSyncStatus: 'pending', + memberSyncLockToken: dispatchToken, + memberSyncLockLeaseAt: now, + updatedAt: now, + }) + .where( + and( + eq(knowledgeConnector.id, connectorId), + eq(knowledgeConnector.accessMode, 'members'), + inArray(knowledgeConnector.status, MEMBER_LOCKABLE_CONNECTOR_STATUSES), + inArray(knowledgeConnector.memberSyncStatus, QUEUEABLE_MEMBER_SYNC_STATUSES), + ...(expectedNextMemberSyncAt + ? [eq(knowledgeConnector.nextMemberSyncAt, expectedNextMemberSyncAt)] + : []), + isNull(knowledgeConnector.memberSyncLockToken), + isNull(knowledgeConnector.syncLockToken), + connectorIsLive() + ) + ) + .returning({ id: knowledgeConnector.id }) + return taken.length > 0 ? dispatchToken : null +} + +async function describeUnacceptedMemberSync( + connectorId: string, + expectedNextMemberSyncAt: Date | undefined +): Promise { + const [row] = await db + .select({ + accessMode: knowledgeConnector.accessMode, + status: knowledgeConnector.status, + memberSyncStatus: knowledgeConnector.memberSyncStatus, + nextMemberSyncAt: knowledgeConnector.nextMemberSyncAt, + syncLockToken: knowledgeConnector.syncLockToken, + archivedAt: knowledgeConnector.archivedAt, + deletedAt: knowledgeConnector.deletedAt, + }) + .from(knowledgeConnector) + .where(eq(knowledgeConnector.id, connectorId)) + .limit(1) + if (!row) return 'Connector no longer exists' + if (row.archivedAt || row.deletedAt) return 'Connector has been archived or deleted' + if (row.accessMode !== 'members') return 'Connector no longer syncs per member' + if (!MEMBER_LOCKABLE_CONNECTOR_STATUSES.some((status) => status === row.status)) { + return `Connector is ${row.status} and is not synced` + } + if (row.syncLockToken) return 'A workspace sync is still running for this connector' + if (row.memberSyncStatus === 'disabled') return 'Member sync is disabled for this connector' + if ( + expectedNextMemberSyncAt && + row.nextMemberSyncAt?.getTime() !== expectedNextMemberSyncAt.getTime() + ) { + return 'The member sync schedule changed after this run was scheduled' + } + return 'A member sync is already queued or running for this connector' +} + +/** + * Releases a queued member sync whose hand-off threw. Guarded on this + * dispatch's own token so a late failure can never clear a replacement's + * entry, and deliberately not laddered: the queue threw, not the connector. + */ +async function releaseFailedMemberDispatch( + connectorId: string, + dispatchToken: string, + error: unknown +): Promise { + const now = new Date() + try { + await db + .update(knowledgeConnector) + .set({ + memberSyncStatus: 'error', + lastMemberSyncError: SYNC_DISPATCH_FAILED_ERROR, + nextMemberSyncAt: now, + memberSyncLockToken: null, + memberSyncLockLeaseAt: null, + updatedAt: now, + }) + .where( + and( + eq(knowledgeConnector.id, connectorId), + eq(knowledgeConnector.memberSyncStatus, 'pending'), + eq(knowledgeConnector.memberSyncLockToken, dispatchToken), + connectorIsLive() + ) + ) + } catch (releaseError) { + logger.error('Failed to release a connector whose member sync dispatch failed', { + connectorId, + dispatchError: toError(error).message, + releaseError: toError(releaseError).message, + }) + } +} + +/** Dispatches one members-mode run with billing attribution fixed by the caller. */ +export async function dispatchMemberSync( + connectorId: string, + options: DispatchMemberSyncOptions +): Promise { + if (!isNonEmptyString(connectorId)) { + throw new Error('Member sync dispatch requires a connector ID') + } + if ( + options.requireRunnable && + (!(options.expectedNextMemberSyncAt instanceof Date) || + Number.isNaN(options.expectedNextMemberSyncAt.getTime())) + ) { + throw new Error('Automatic member sync dispatch requires the expected next sync time') + } + + const requestId = options.requestId ?? generateId() + const payload = assertMemberSyncPayload({ + connectorId, + requestId, + billingAttribution: options.billingAttribution, + }) + + const [row] = await db + .select({ + knowledgeBaseId: knowledgeConnector.knowledgeBaseId, + accessMode: knowledgeConnector.accessMode, + status: knowledgeConnector.status, + memberSyncStatus: knowledgeConnector.memberSyncStatus, + nextMemberSyncAt: knowledgeConnector.nextMemberSyncAt, + archivedAt: knowledgeConnector.archivedAt, + deletedAt: knowledgeConnector.deletedAt, + workspaceId: knowledgeBase.workspaceId, + kbDeletedAt: knowledgeBase.deletedAt, + }) + .from(knowledgeConnector) + .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) + .where(eq(knowledgeConnector.id, connectorId)) + .limit(1) + + if (!row) { + logger.warn('Skipping member sync dispatch: connector not found', { connectorId, requestId }) + return { queued: false, reason: 'Connector no longer exists' } + } + if (row.kbDeletedAt) { + logger.warn('Skipping member sync dispatch: knowledge base is deleted', { + connectorId, + knowledgeBaseId: row.knowledgeBaseId, + requestId, + }) + await db + .update(knowledgeConnector) + .set({ + memberSyncStatus: 'error', + nextMemberSyncAt: null, + lastMemberSyncError: 'Knowledge base deleted', + memberSyncLockToken: null, + memberSyncLockLeaseAt: null, + updatedAt: new Date(), + }) + .where(eq(knowledgeConnector.id, connectorId)) + return { queued: false, reason: 'Knowledge base has been deleted' } + } + if (row.archivedAt || row.deletedAt) { + return { queued: false, reason: 'Connector has been archived or deleted' } + } + if (row.accessMode !== 'members') { + return { queued: false, reason: 'Connector does not sync per member' } + } + if (options.requireRunnable && row.status !== 'active' && row.status !== 'error') { + return { + queued: false, + reason: `Connector is ${row.status} and is not synced automatically`, + } + } + if ( + payload.dispatchToken === undefined && + options.requireRunnable && + !QUEUEABLE_MEMBER_SYNC_STATUSES.some((status) => status === row.memberSyncStatus) + ) { + return { + queued: false, + reason: `Member sync is ${row.memberSyncStatus} and is not run automatically`, + } + } + if ( + options.expectedNextMemberSyncAt && + row.nextMemberSyncAt?.getTime() !== options.expectedNextMemberSyncAt.getTime() + ) { + return { + queued: false, + reason: 'The member sync schedule changed after this run was scheduled', + } + } + if (!row.workspaceId) { + throw new Error(`Connector ${connectorId} is missing workspace billing context`) + } + if (payload.billingAttribution.workspaceId !== row.workspaceId) { + throw new Error( + `Member sync billing attribution does not match connector workspace ${row.workspaceId}` + ) + } + + const dispatchToken = await markMemberSyncPending(connectorId, options.expectedNextMemberSyncAt) + if (!dispatchToken) { + const reason = await describeUnacceptedMemberSync(connectorId, options.expectedNextMemberSyncAt) + logger.info('Skipping member sync dispatch: connector is not accepting a queued run', { + connectorId, + reason, + requestId, + }) + return { queued: false, reason } + } + + if (isTriggerAvailable()) { + try { + const idempotencyKey = options.expectedNextMemberSyncAt + ? await idempotencyKeys.create( + `${MEMBER_SYNC_TASK_ID}:${connectorId}:${options.expectedNextMemberSyncAt.toISOString()}`, + { scope: 'global' } + ) + : undefined + await tasks.trigger( + MEMBER_SYNC_TASK_ID, + { ...payload, dispatchToken }, + { + ...(idempotencyKey ? { idempotencyKey } : {}), + tags: [ + `connectorId:${connectorId}`, + `knowledgeBaseId:${row.knowledgeBaseId}`, + `workspaceId:${row.workspaceId}`, + `userId:${payload.billingAttribution.actorUserId}`, + ], + region: await resolveTriggerRegion(), + } + ) + } catch (error) { + await releaseFailedMemberDispatch(connectorId, dispatchToken, error) + throw error + } + logger.info('Dispatched member sync to Trigger.dev', { connectorId, requestId }) + return { queued: true } + } + + executeMemberSync(connectorId, { + billingAttribution: payload.billingAttribution, + dispatchToken, + }).catch(async (error) => { + logger.error(`Member sync failed for connector ${connectorId}`, { + error: toError(error).message, + requestId, + }) + await releaseFailedMemberDispatch(connectorId, dispatchToken, error) + }) + return { queued: true } +} + +/** + * Queues a member run for every connector that crawls through the option a + * member just connected, so their documents arrive within minutes rather + * than at the next scheduled run. Best effort: a refused or failed dispatch is + * logged, the remaining connectors are still queued, and the schedule catches + * up on whichever was not. + */ +export async function dispatchMemberSyncsForCredentialOption(input: { + workspaceId: string + credentialGroupOptionId: string +}): Promise { + const connectors = await db + .select({ id: knowledgeConnector.id }) + .from(knowledgeConnector) + .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) + .where( + and( + eq(knowledgeBase.workspaceId, input.workspaceId), + isNull(knowledgeBase.deletedAt), + eq(knowledgeConnector.accessMode, 'members'), + eq(knowledgeConnector.credentialGroupOptionId, input.credentialGroupOptionId), + inArray(knowledgeConnector.status, ['active', 'error']), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + if (connectors.length === 0) return + const billingAttribution = await resolveSystemBillingAttribution(input.workspaceId) + for (const connector of connectors) { + try { + const dispatch = await dispatchMemberSync(connector.id, { billingAttribution }) + if (!dispatch.queued) { + logger.info('Member sync after a member connected was not queued', { + connectorId: connector.id, + reason: dispatch.reason, + }) + } + } catch (error) { + logger.warn('Member sync after a member connected could not be dispatched', { + connectorId: connector.id, + error: toError(error).message, + }) + } + } +} diff --git a/apps/sim/lib/knowledge/connectors/member-sync-engine.test.ts b/apps/sim/lib/knowledge/connectors/member-sync-engine.test.ts new file mode 100644 index 00000000000..ba573316dab --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/member-sync-engine.test.ts @@ -0,0 +1,293 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('@/connectors/registry.server', () => ({ CONNECTOR_REGISTRY: {} })) +vi.mock('@/lib/knowledge/documents/service', () => ({ + hardDeleteDocuments: vi.fn(), + processDocumentsWithQueue: vi.fn(), + ConnectorSyncDeletionGuardError: class ConnectorSyncDeletionGuardError extends Error {}, +})) +vi.mock('@/lib/knowledge/connectors/member-access', () => ({ + KnowledgeConnectorMemberAccessDeniedError: class extends Error {}, + listKnowledgeConnectorMemberCredentials: vi.fn(), + mintKnowledgeConnectorMemberToken: vi.fn(), +})) +vi.mock('@/lib/billing/core/workspace-access', () => ({ + getWorkspaceOwnerSubscriptionAccess: vi.fn(), +})) +vi.mock('@/lib/credential-groups/availability', () => ({ isCredentialGroupsAvailable: vi.fn() })) + +import { + admitMemberListing, + buildMemberSyncFailureUpdate, + deriveMemberActive, + memberFailureBackoffMs, + memberNextAttemptAt, + nextMemberSyncTime, + shouldListFully, +} from '@/lib/knowledge/connectors/member-sync-engine' +import { + CONNECTOR_AUTO_DISABLED_ERROR, + MAX_CONSECUTIVE_FAILURES, + MEMBER_CHANGE_FEED_FULL_RECRAWL_MINUTES, + MEMBER_FULL_RECRAWL_MINUTES, +} from '@/lib/knowledge/connectors/sync-limits' +import { + CONNECTOR_SYNC_MAX_CORPUS_DOCUMENTS, + runChangeFeedPass, +} from '@/lib/knowledge/connectors/sync-primitives' +import type { ExternalChangeList, ExternalDocument } from '@/connectors/types' + +function doc(externalId: string, content = 'x'): ExternalDocument { + return { externalId, title: externalId, content, mimeType: 'text/plain', metadata: {} } +} + +describe('member sync engine decisions', () => { + describe('deriveMemberActive', () => { + const live = { groupActive: true, optionActive: true } + + it.each([ + ['active credential in a live enrollment', 'active', 'completed', live, true], + ['active credential mid-enrollment', 'active', 'in_progress', live, true], + ['credential needing re-auth', 'needs_reauth', 'completed', live, false], + ['revoked credential', 'revoked', 'completed', live, false], + ['revoked enrollment', 'active', 'revoked', live, false], + ['invited-only enrollment', 'active', 'invited', live, false], + ['disabled option', 'active', 'completed', { groupActive: true, optionActive: false }, false], + ['disabled group', 'active', 'completed', { groupActive: false, optionActive: true }, false], + ] as const)('%s', (_name, managedOauthStatus, enrollmentStatus, option, expected) => { + expect(deriveMemberActive({ managedOauthStatus, enrollmentStatus }, option)).toBe(expected) + }) + }) + + describe('shouldListFully', () => { + const now = new Date('2026-09-01T12:00:00Z') + + it('lists fully before any complete listing exists', () => { + expect(shouldListFully(null, null, now)).toBe(true) + expect(shouldListFully(now, null, now)).toBe(true) + expect(shouldListFully(null, now, now)).toBe(true) + }) + + it('lists incrementally inside the recrawl window and fully once it elapses', () => { + const windowMs = MEMBER_FULL_RECRAWL_MINUTES * 60 * 1000 + const recent = new Date(now.getTime() - windowMs + 60_000) + const stale = new Date(now.getTime() - windowMs) + expect(shouldListFully(recent, recent, now)).toBe(false) + expect(shouldListFully(stale, stale, now)).toBe(true) + }) + + it('stretches the window for a member whose change feed is open', () => { + const feedWindowMs = MEMBER_CHANGE_FEED_FULL_RECRAWL_MINUTES * 60 * 1000 + const beyondPlainWindow = new Date(now.getTime() - MEMBER_FULL_RECRAWL_MINUTES * 60 * 1000) + expect(MEMBER_CHANGE_FEED_FULL_RECRAWL_MINUTES).toBeGreaterThan(MEMBER_FULL_RECRAWL_MINUTES) + expect( + shouldListFully( + beyondPlainWindow, + beyondPlainWindow, + now, + MEMBER_CHANGE_FEED_FULL_RECRAWL_MINUTES + ) + ).toBe(false) + const stale = new Date(now.getTime() - feedWindowMs) + expect(shouldListFully(stale, stale, now, MEMBER_CHANGE_FEED_FULL_RECRAWL_MINUTES)).toBe(true) + }) + }) + + describe('runChangeFeedPass', () => { + function pass( + pages: ExternalChangeList[], + options: { deadlineAt?: number; maxPages?: number } = {} + ) { + const listChanges = vi.fn(async (_token: string, _config: unknown, cursor: string) => { + const page = pages[Number(cursor.replace('c', ''))] + if (!page) throw new Error(`no page for ${cursor}`) + return page + }) + return { + listChanges, + run: () => + runChangeFeedPass({ + connectorId: 'c-1', + connectorConfig: { listChanges }, + sourceConfig: {}, + syncContext: {}, + cursor: 'c0', + beforePage: async () => undefined, + getAccessToken: async () => 'token', + ...options, + }), + } + } + + it('keeps the last word on each item and resumes past the drained feed', async () => { + const feed = pass([ + { + changes: [ + { kind: 'upsert', externalId: 'a', document: doc('a', 'v1') }, + { kind: 'removed', externalId: 'b' }, + ], + nextCursor: 'c1', + hasMore: true, + }, + { + changes: [ + { kind: 'removed', externalId: 'a' }, + { kind: 'upsert', externalId: 'b', document: doc('b') }, + { kind: 'upsert', externalId: 'c', document: doc('c') }, + ], + nextCursor: 'resume', + hasMore: false, + }, + ]) + const result = await feed.run() + + expect(result.upserts.map((d) => d.externalId)).toEqual(['b', 'c']) + expect(result.removedExternalIds).toEqual(['a']) + expect(result.cursor).toBe('resume') + expect(result.exhausted).toBe(true) + expect(result.budgetAborted).toBe(false) + expect(feed.listChanges).toHaveBeenCalledTimes(2) + }) + + it('stops at the page cap with the cursor past the pages it read', async () => { + const feed = pass( + [ + { changes: [{ kind: 'removed', externalId: 'x' }], nextCursor: 'c1', hasMore: true }, + { changes: [], nextCursor: 'c2', hasMore: true }, + ], + { maxPages: 1 } + ) + const result = await feed.run() + + expect(result.removedExternalIds).toEqual(['x']) + expect(result.cursor).toBe('c1') + expect(result.exhausted).toBe(false) + expect(result.budgetAborted).toBe(false) + }) + + it('reads nothing past the deadline and leaves the cursor where it was', async () => { + const feed = pass([{ changes: [], nextCursor: 'c1', hasMore: false }], { + deadlineAt: Date.now() - 1, + }) + const result = await feed.run() + + expect(result.cursor).toBe('c0') + expect(result.budgetAborted).toBe(true) + expect(result.exhausted).toBe(false) + expect(feed.listChanges).not.toHaveBeenCalled() + }) + }) + + describe('memberNextAttemptAt', () => { + const now = new Date('2026-09-01T12:00:00Z') + + it('is exactly one interval on, with no jitter, so the connector run finds the member due', () => { + expect(memberNextAttemptAt(now, 60)).toEqual(new Date('2026-09-01T13:00:00Z')) + }) + + it('waits for the next manual run on a manual-only connector', () => { + expect(memberNextAttemptAt(now, 0)).toBeNull() + }) + }) + + describe('memberFailureBackoffMs', () => { + it('doubles on the connector interval and caps at a day', () => { + expect(memberFailureBackoffMs(1, 60)).toBe(60 * 60 * 1000) + expect(memberFailureBackoffMs(2, 60)).toBe(2 * 60 * 60 * 1000) + expect(memberFailureBackoffMs(3, 60)).toBe(4 * 60 * 60 * 1000) + expect(memberFailureBackoffMs(10, 60)).toBe(24 * 60 * 60 * 1000) + expect(memberFailureBackoffMs(40, 60)).toBe(24 * 60 * 60 * 1000) + }) + + it('paces a manual-only connector on an hourly base', () => { + expect(memberFailureBackoffMs(1, 0)).toBe(60 * 60 * 1000) + }) + }) + + describe('buildMemberSyncFailureUpdate', () => { + const now = new Date('2026-09-01T12:00:00Z') + + it('re-enters the shared ladder over the member columns and releases the lease', () => { + const update = buildMemberSyncFailureUpdate(now, 0, 'boom') + expect(update.memberSyncStatus).toBe('error') + expect(update.lastMemberSyncError).toBe('boom') + expect(update.memberSyncConsecutiveFailures).toBe(1) + expect(update.nextMemberSyncAt?.getTime()).toBeGreaterThan(now.getTime()) + expect(update.memberSyncLockToken).toBeNull() + expect(update.memberSyncLockLeaseAt).toBeNull() + }) + + it('disables after the shared threshold with the shared message', () => { + const update = buildMemberSyncFailureUpdate(now, MAX_CONSECUTIVE_FAILURES - 1, 'boom') + expect(update.memberSyncStatus).toBe('disabled') + expect(update.lastMemberSyncError).toBe(CONNECTOR_AUTO_DISABLED_ERROR) + expect(update.nextMemberSyncAt).toBeNull() + }) + + it('honours a longer provider retry hint but never a shorter one', () => { + const ladder = buildMemberSyncFailureUpdate(now, 0, 'boom').nextMemberSyncAt!.getTime() + const longer = buildMemberSyncFailureUpdate(now, 0, 'boom', 6 * 60 * 60 * 1000) + const shorter = buildMemberSyncFailureUpdate(now, 0, 'boom', 1000) + expect(longer.nextMemberSyncAt!.getTime()).toBe(now.getTime() + 6 * 60 * 60 * 1000) + expect(shorter.nextMemberSyncAt!.getTime()).toBe(ladder) + }) + }) + + describe('nextMemberSyncTime', () => { + const now = new Date('2026-09-01T12:00:00Z') + + it('re-dispatches immediately while members remain due', () => { + expect(nextMemberSyncTime(now, 1440, true)).toEqual(now) + expect(nextMemberSyncTime(now, 0, true)).toEqual(now) + }) + + it('schedules the interval plus bounded jitter, or nothing for a manual connector', () => { + const next = nextMemberSyncTime(now, 60, false)! + expect(next.getTime()).toBeGreaterThanOrEqual(now.getTime() + 60 * 60 * 1000) + expect(next.getTime()).toBeLessThanOrEqual(now.getTime() + 60 * 60 * 1000 + 300_000) + expect(nextMemberSyncTime(now, 0, false)).toBeNull() + }) + }) + + describe('admitMemberListing', () => { + it('keeps the first writer and records every observer', () => { + const union = new Map() + const first = admitMemberListing(union, 'm-1', [doc('a', 'first'), doc('b')], 'c-1', 0) + const second = admitMemberListing( + union, + 'm-2', + [doc('a', 'second'), doc('c')], + 'c-1', + first.retainedBytes + ) + + expect([...first.seenExternalIds]).toEqual(['a', 'b']) + expect([...second.seenExternalIds]).toEqual(['a', 'c']) + expect(union.get('a')).toEqual({ document: doc('a', 'first'), observers: ['m-1', 'm-2'] }) + expect(union.get('b')?.observers).toEqual(['m-1']) + expect(union.get('c')?.observers).toEqual(['m-2']) + expect(second.retainedBytes).toBeGreaterThan(first.retainedBytes) + }) + + it('counts a member once per external id even when their listing repeats it', () => { + const union = new Map() + const admitted = admitMemberListing(union, 'm-1', [doc('a'), doc('a')], 'c-1', 0) + expect(admitted.seenExternalIds.size).toBe(1) + expect(union.get('a')?.observers).toEqual(['m-1']) + }) + + it('holds the union to the working-set ceiling', () => { + const union = new Map() + const documents = Array.from({ length: CONNECTOR_SYNC_MAX_CORPUS_DOCUMENTS }, (_, index) => + doc(`d-${index}`) + ) + admitMemberListing(union, 'm-1', documents, 'c-1', 0) + expect(() => admitMemberListing(union, 'm-2', [doc('overflow')], 'c-1', 0)).toThrow( + 'exceeds the safe per-corpus limit' + ) + }) + }) +}) diff --git a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts new file mode 100644 index 00000000000..afda33d905f --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts @@ -0,0 +1,1698 @@ +import { db } from '@sim/db' +import { + credential, + document, + knowledgeBase, + knowledgeConnector, + knowledgeConnectorMember, + knowledgeConnectorMemberSyncLog, +} from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { getErrorMessage, toError } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { randomInt } from '@sim/utils/random' +import { and, eq, inArray, isNull, lte, notExists, sql } from 'drizzle-orm' +import { LRUCache } from 'lru-cache' +import { + assertBillingAttributionSnapshot, + type BillingAttributionSnapshot, +} from '@/lib/billing/core/billing-attribution' +import { + type CredentialGroupOptionCredentialReference, + isManagedCredentialGroupBindingLive, + loadCredentialGroupCredentialListContext, +} from '@/lib/credential-groups/credentials' +import type { DbOrTx } from '@/lib/db/types' +import { isKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' +import { EMPTY_ACL, subjectToken } from '@/lib/knowledge/access/tokens' +import { + KnowledgeConnectorMemberAccessDeniedError, + listKnowledgeConnectorMemberCredentials, + mintKnowledgeConnectorMemberToken, +} from '@/lib/knowledge/connectors/member-access' +import { + applyMemberDocumentLifecycle, + listObservedDocumentIds, + materializeDocumentAcls, + recordMemberObservations, + removeMemberObservationsForDocuments, + removeUnseenMemberObservations, + rewriteConnectorAcls, +} from '@/lib/knowledge/connectors/member-observations' +import { inviteWorkspaceMembersToCredentialGroup } from '@/lib/knowledge/connectors/member-provisioning' +import { + CONNECTOR_AUTO_DISABLED_ERROR, + CONNECTOR_FAILURE_BACKOFF_CAP_MINUTES, + connectorFailureBackoffMinutes, + MAX_CONSECUTIVE_FAILURES, + MEMBER_CHANGE_FEED_FULL_RECRAWL_MINUTES, + MEMBER_FULL_RECRAWL_MINUTES, + MEMBER_SUSPENDED_PURGE_DAYS, + MEMBER_SYNC_MAX_PAGES_PER_MEMBER, + MEMBER_SYNC_SOFT_BUDGET_SECONDS, +} from '@/lib/knowledge/connectors/sync-limits' +import { + assertSyncLeaseHeldInTx, + createMemberSyncLease, + holdsMemberSyncLockToken, + MEMBER_LOCKABLE_CONNECTOR_STATUSES, + SyncLockLostException, + stillHoldsMemberSyncLock, +} from '@/lib/knowledge/connectors/sync-lock' +import type { KnowledgeBaseOwner } from '@/lib/knowledge/connectors/sync-persistence' +import { + addSourcePagePayloadBytes, + ConnectorDeletedException, + ConnectorSyncCapacityError, + ConnectorSyncWorkingSetLimitError, + classifyListing, + classifySuspectListing, + createSyncRunState, + loadOwnedCorpus, + processDocOps, + RETRY_WINDOW_DAYS, + runChangeFeedPass, + runListingPass, + sourcePageFitsSyncWorkingSet, + sweepStuckDocuments, + syncWorkingSetQueryLimit, +} from '@/lib/knowledge/connectors/sync-primitives' +import { getRetryAfterMs, isRateLimitError } from '@/lib/knowledge/documents/utils' +import { CONNECTOR_REGISTRY } from '@/connectors/registry.server' +import type { + ConnectorConfig, + ExternalDocument, + SyncResult, + SyncSkipReason, +} from '@/connectors/types' +import { PER_MEMBER_LISTING_CONTEXT } from '@/connectors/utils' + +const logger = createLogger('ConnectorMemberSyncEngine') + +/** Observers tried, in listing order, before a document's hydration is given up on. */ +const HYDRATION_OBSERVER_ATTEMPTS = 3 +/** A minted token is reused for this long before the member is re-minted. */ +const MEMBER_TOKEN_REUSE_MS = 45 * 60 * 1000 +/** Members whose tokens one run keeps at once; a memory backstop, not a working-set limit. */ +const MEMBER_TOKEN_CACHE_MAX = 10_000 +/** Overlap subtracted from a member's incremental watermark, covering source clock skew. */ +const INCREMENTAL_OVERLAP_MS = 5 * 60 * 1000 +/** Member rows read per page while reconciling membership. */ +const MEMBER_CREDENTIAL_PAGE_SIZE = 100 +/** Backoff ceiling for one member's failure ladder. */ +const MEMBER_BACKOFF_CAP_MS = 24 * 60 * 60 * 1000 +/** Interval a manual-only connector (interval 0) uses to pace member retries. */ +const MEMBER_BACKOFF_BASE_MINUTES = 60 + +export interface MemberSyncResult extends SyncResult { + membersClaimed: number + membersCompleted: number + membersIncomplete: number + membersFailed: number + /** Whether members were still due when the run's budget ended. */ + membersRemaining: boolean + docsListed: number + docsHydratedOnce: number + observationsAdded: number + observationsRemoved: number + docsTombstoned: number + docsResurrected: number + docsPurged: number + credentialsAudited: number +} + +export interface ExecuteMemberSyncOptions { + billingAttribution: BillingAttributionSnapshot + /** The queue entry this run is allowed to consume; see `MemberSyncPayload.dispatchToken`. */ + dispatchToken?: string +} + +type MemberRow = typeof knowledgeConnectorMember.$inferSelect + +/** One member's credential as the option reports it, with the membership state it implies. */ +export interface MemberCredentialSnapshot { + credentialId: string + subjectToken: string + active: boolean +} + +/** + * How a member's view of the source was read this run. A full listing is the + * only kind that can withdraw access by omission; the change feed withdraws it + * by an explicit removal; an incremental listing refreshes content only. + */ +type MemberListingMode = 'full' | 'changes' | 'incremental' + +/** What one member's listing established for this run. */ +interface MemberListingOutcome { + member: MemberRow + mode: MemberListingMode + listingStartedAt: Date + seenExternalIds: Set + /** Items the change feed reported as deleted or no longer reachable by the member. */ + removedExternalIds: readonly string[] + listedCount: number + complete: boolean + /** + * An incomplete listing the next run can pick up where this one stopped — + * the budget ended, or a feed pass hit its page cap — rather than one a + * retry cannot improve on, such as a capped or truncated source. + */ + resumable: boolean + /** + * The member was the run's only claim and still ran out of budget: a listing + * no run can finish alone, so it backs off instead of re-dispatching forever. + */ + exhaustedRunAlone: boolean + suspect: boolean + /** Cursor to store when this outcome lands: a value, null to close the feed, undefined to leave it. */ + changeCursor: string | null | undefined +} + +interface UnionEntry { + document: ExternalDocument + /** Member ids whose listings returned the document, in listing order. */ + observers: string[] +} + +function emptyResult(): MemberSyncResult { + return { + docsAdded: 0, + docsUpdated: 0, + docsDeleted: 0, + docsUnchanged: 0, + docsSkipped: 0, + docsFailed: 0, + processingDispatch: { requested: 0, accepted: 0, failed: 0 }, + membersClaimed: 0, + membersCompleted: 0, + membersIncomplete: 0, + membersFailed: 0, + membersRemaining: false, + docsListed: 0, + docsHydratedOnce: 0, + observationsAdded: 0, + observationsRemoved: 0, + docsTombstoned: 0, + docsResurrected: 0, + docsPurged: 0, + credentialsAudited: 0, + } +} + +function skipped(result: MemberSyncResult, skipReason: SyncSkipReason): MemberSyncResult { + return { ...result, skipReason } +} + +/** + * Whether a credential collected under the option currently makes its owner an + * active member: the credential is usable, the enrollment is live, and the + * option and group are still active. Anything else suspends the member, which + * drops their token from every ACL but keeps their observations. + */ +export function deriveMemberActive( + credential: Pick< + CredentialGroupOptionCredentialReference, + 'managedOauthStatus' | 'enrollmentStatus' + >, + option: { groupActive: boolean; optionActive: boolean } +): boolean { + return isManagedCredentialGroupBindingLive({ + managedOauthStatus: credential.managedOauthStatus, + enrollmentStatus: credential.enrollmentStatus, + groupStatus: option.groupActive ? 'active' : 'disabled', + optionStatus: option.optionActive ? 'active' : 'disabled', + }) +} + +/** + * Whether a member needs a full listing this run. Without a change feed only a + * full listing grants or removes access, so every member gets one at least + * every {@link MEMBER_FULL_RECRAWL_MINUTES} and an incremental listing + * refreshes content between them. A member whose feed is open needs one only + * every {@link MEMBER_CHANGE_FEED_FULL_RECRAWL_MINUTES}, as a check that the + * feed missed nothing. + */ +export function shouldListFully( + memberSyncedThrough: Date | null, + lastCompleteListingAt: Date | null, + now: Date, + recrawlMinutes: number = MEMBER_FULL_RECRAWL_MINUTES +): boolean { + if (!memberSyncedThrough || !lastCompleteListingAt) return true + return now.getTime() - lastCompleteListingAt.getTime() >= recrawlMinutes * 60 * 1000 +} + +/** Whether a connector can keep a per-member change feed at all. */ +function supportsChangeFeed( + connectorConfig: ConnectorConfig +): connectorConfig is ConnectorConfig & { + listChanges: NonNullable + getChangeCursor: NonNullable +} { + return ( + typeof connectorConfig.listChanges === 'function' && + typeof connectorConfig.getChangeCursor === 'function' + ) +} + +/** The next attempt for a member whose listing threw: exponential on the connector's interval, capped at a day. */ +export function memberFailureBackoffMs(failures: number, syncIntervalMinutes: number): number { + const baseMinutes = syncIntervalMinutes > 0 ? syncIntervalMinutes : MEMBER_BACKOFF_BASE_MINUTES + const exponent = Math.min(Math.max(failures, 1) - 1, 20) + return Math.min(2 ** exponent * baseMinutes * 60 * 1000, MEMBER_BACKOFF_CAP_MS) +} + +/** + * The connector row a failed run writes: the content engine's ladder over the + * member columns, so a connector that keeps failing per member backs off and + * eventually disables exactly as a workspace-mode one does. + */ +export function buildMemberSyncFailureUpdate( + now: Date, + previousFailures: number | null | undefined, + errorMessage: string, + retryAfterMs?: number +) { + const failures = (previousFailures ?? 0) + 1 + const disabled = failures >= MAX_CONSECUTIVE_FAILURES + const failureBackoffMs = connectorFailureBackoffMinutes(failures) * 60 * 1000 + const maximumBackoffMs = CONNECTOR_FAILURE_BACKOFF_CAP_MINUTES * 60 * 1000 + const providerBackoffMs = + typeof retryAfterMs === 'number' && Number.isFinite(retryAfterMs) && retryAfterMs > 0 + ? Math.min(retryAfterMs, maximumBackoffMs) + : 0 + return { + memberSyncStatus: (disabled ? 'disabled' : 'error') as 'disabled' | 'error', + lastMemberSyncError: disabled ? CONNECTOR_AUTO_DISABLED_ERROR : errorMessage, + nextMemberSyncAt: disabled + ? null + : new Date(now.getTime() + Math.max(failureBackoffMs, providerBackoffMs)), + memberSyncConsecutiveFailures: failures, + memberSyncLockToken: null, + memberSyncLockLeaseAt: null, + updatedAt: now, + } +} + +/** + * When a member who completed is next due: exactly one interval on, with no + * jitter, so they are due whenever the connector's own (jittered) run lands. + * Null on a manual-only connector: with its next manual run. + */ +export function memberNextAttemptAt(now: Date, syncIntervalMinutes: number): Date | null { + return syncIntervalMinutes > 0 ? new Date(now.getTime() + syncIntervalMinutes * 60_000) : null +} + +/** The next scheduled run: immediately while members remain due, else the interval plus jitter. */ +export function nextMemberSyncTime( + now: Date, + syncIntervalMinutes: number, + membersRemaining: boolean +): Date | null { + if (membersRemaining) return now + if (syncIntervalMinutes <= 0) return null + const jitterMs = randomInt(0, Math.min(syncIntervalMinutes * 6_000, 300_000)) + return new Date(now.getTime() + syncIntervalMinutes * 60_000 + jitterMs) +} + +/** + * Admits one member's listing into the run's union: first writer wins on a + * repeated external id, every writer is recorded as an observer, and the + * union is held to the same working-set and payload limits as a single + * workspace-mode listing. + */ +export function admitMemberListing( + union: Map, + memberId: string, + documents: readonly ExternalDocument[], + connectorId: string, + retainedBytes: number +): { seenExternalIds: Set; retainedBytes: number } { + const seenExternalIds = new Set() + const admitted: ExternalDocument[] = [] + for (const doc of documents) { + if (seenExternalIds.has(doc.externalId)) continue + seenExternalIds.add(doc.externalId) + const existing = union.get(doc.externalId) + if (existing) { + existing.observers.push(memberId) + continue + } + admitted.push(doc) + } + if (!sourcePageFitsSyncWorkingSet(union.size, admitted.length)) { + throw new ConnectorSyncWorkingSetLimitError(connectorId, 'source listing') + } + const nextBytes = addSourcePagePayloadBytes(retainedBytes, admitted) + for (const doc of admitted) { + union.set(doc.externalId, { document: doc, observers: [memberId] }) + } + return { seenExternalIds, retainedBytes: nextBytes } +} + +interface MemberSyncRun { + connectorId: string + knowledgeBaseId: string + workspaceId: string + runId: string + runStartedAt: Date + deadlineAt: number + result: MemberSyncResult + lease: ReturnType +} + +/** A token minted for a member, reused within the run until it ages out. */ +interface MemberTokenCache { + get(memberId: string): Promise +} + +function createMemberTokenCache(input: { + run: MemberSyncRun + connectorConfig: Pick + credentialIdByMemberId: Map +}): MemberTokenCache { + const { auth } = input.connectorConfig + if (auth.mode !== 'oauth') throw new Error('Members mode requires an OAuth connector') + const tokens = new LRUCache({ + max: MEMBER_TOKEN_CACHE_MAX, + ttl: MEMBER_TOKEN_REUSE_MS, + fetchMethod: async (memberId) => { + const credentialId = input.credentialIdByMemberId.get(memberId) + if (!credentialId) throw new Error(`Member ${memberId} has no credential in this run`) + const minted = await mintKnowledgeConnectorMemberToken({ + connectorId: input.run.connectorId, + workspaceId: input.run.workspaceId, + credentialId, + expectedProviderId: auth.provider, + requiredScopes: auth.requiredScopes ?? [], + runId: input.run.runId, + }) + input.run.result.credentialsAudited += 1 + return minted.accessToken + }, + }) + return { + async get(memberId) { + const accessToken = await tokens.fetch(memberId) + if (!accessToken) throw new Error(`No token could be minted for member ${memberId}`) + return accessToken + }, + } +} + +/** + * Runs `fn` in a transaction that first proves this run still holds the + * connector's member lease, taking the connector row's lock so the scheduler + * cannot reclaim the lease mid-transaction. A run that stalled past the lease + * TTL and resumed after a replacement took over therefore never lands its + * observations or ACLs over the replacement's; it ends as superseded. + */ +async function withMemberLease( + run: Pick, + fn: (tx: DbOrTx) => Promise +): Promise { + return db.transaction(async (tx) => { + const [held] = await tx + .select({ id: knowledgeConnector.id }) + .from(knowledgeConnector) + .where(stillHoldsMemberSyncLock(run.connectorId, run.runId)) + .for('update') + if (!held) throw new SyncLockLostException(run.connectorId) + return fn(tx) + }) +} + +async function acquireMemberSyncLock( + connectorId: string, + runId: string, + dispatchToken: string | undefined +): Promise { + const now = new Date() + const [row] = await db + .update(knowledgeConnector) + .set({ + memberSyncStatus: 'running', + memberSyncLockToken: runId, + memberSyncLockLeaseAt: now, + updatedAt: now, + }) + .where( + and( + eq(knowledgeConnector.id, connectorId), + eq(knowledgeConnector.accessMode, 'members'), + inArray(knowledgeConnector.status, MEMBER_LOCKABLE_CONNECTOR_STATUSES), + inArray(knowledgeConnector.memberSyncStatus, ['idle', 'pending', 'error']), + ...(dispatchToken ? [eq(knowledgeConnector.memberSyncLockToken, dispatchToken)] : []), + isNull(knowledgeConnector.syncLockToken), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + .returning() + return row ?? null +} + +async function insertMemberSyncLog(runId: string, connectorId: string, startedAt: Date) { + await db.insert(knowledgeConnectorMemberSyncLog).values({ + id: runId, + connectorId, + status: 'started', + startedAt, + }) +} + +/** + * Finishes an ACL rewrite a mode switch left behind before this run lists + * anything: every document of the connector is hidden until an observation + * makes it visible again. Bounded by the run's own budget like every other + * step, so a large corpus is hidden across as many runs as it takes rather + * than one run that never reaches a member; returns whether it finished. + */ +async function finishPendingAccessRewrite(run: MemberSyncRun): Promise { + const finished = await rewriteConnectorAcls(run.connectorId, EMPTY_ACL, { + deadlineAt: run.deadlineAt, + beforeBatch: run.lease.beatIfDue, + lease: run.lease, + }) + if (!finished) return false + await db + .update(knowledgeConnector) + .set({ accessRewritePending: false, updatedAt: new Date() }) + .where( + and( + eq(knowledgeConnector.id, run.connectorId), + stillHoldsMemberSyncLock(run.connectorId, run.runId) + ) + ) + return true +} + +interface MembershipReconciliation { + /** Documents whose ACL must be rematerialised because an observer's state or token changed. */ + affectedDocumentIds: Set +} + +/** The credential-group option the connector was bound to no longer exists. */ +class MemberBindingGoneError extends Error { + constructor(message: string) { + super(message) + this.name = 'MemberBindingGoneError' + } +} + +/** + * Mirrors the credential-group option onto member rows: inserts new + * credentials, moves members between active and suspended, rewrites a subject + * token that changed, drops members whose credential left the option, and + * purges members suspended past the window. Every change that alters what an + * observer contributes to an ACL is collected for rematerialisation. + */ +async function reconcileMembership( + run: MemberSyncRun, + binding: { credentialGroupId: string; credentialGroupOptionId: string } +): Promise { + const group = await loadCredentialGroupCredentialListContext(binding.credentialGroupId) + if (!group) { + throw new MemberBindingGoneError( + 'The Credential Group this connector synced through was deleted' + ) + } + const option = group.options.find((candidate) => candidate.id === binding.credentialGroupOptionId) + if (!option) { + throw new MemberBindingGoneError( + 'The Credential Group option this connector synced through was removed' + ) + } + const optionState = { + groupActive: group.status === 'active', + optionActive: option.status === 'active', + } + + const snapshots = new Map() + let cursor: string | undefined + do { + const page = await listKnowledgeConnectorMemberCredentials({ + workspaceId: run.workspaceId, + credentialGroupId: binding.credentialGroupId, + credentialGroupOptionId: binding.credentialGroupOptionId, + connectorId: run.connectorId, + limit: MEMBER_CREDENTIAL_PAGE_SIZE, + cursor, + }) + for (const credential of page.credentials) { + snapshots.set(credential.credentialId, { + credentialId: credential.credentialId, + subjectToken: subjectToken(credential), + active: deriveMemberActive(credential, optionState), + }) + } + cursor = page.nextCursor ?? undefined + } while (cursor) + + const existing = await db + .select() + .from(knowledgeConnectorMember) + .where(eq(knowledgeConnectorMember.connectorId, run.connectorId)) + const existingByCredential = new Map(existing.map((row) => [row.credentialId, row])) + const now = new Date() + const purgeCutoff = new Date(now.getTime() - MEMBER_SUSPENDED_PURGE_DAYS * 24 * 60 * 60 * 1000) + + const inserts: (typeof knowledgeConnectorMember.$inferInsert)[] = [] + const affectedMemberIds: string[] = [] + const updates: Array<{ + id: string + values: Partial + }> = [] + const deleteMemberIds: string[] = [] + + for (const snapshot of snapshots.values()) { + const row = existingByCredential.get(snapshot.credentialId) + const status = snapshot.active ? 'active' : 'suspended' + if (!row) { + inserts.push({ + id: generateId(), + workspaceId: run.workspaceId, + connectorId: run.connectorId, + credentialId: snapshot.credentialId, + subjectToken: snapshot.subjectToken, + status, + suspendedAt: snapshot.active ? null : now, + /** Due now, so a run that cannot reach everyone re-dispatches until it has. */ + nextAttemptAt: now, + createdAt: now, + updatedAt: now, + }) + continue + } + if ( + row.status === 'suspended' && + !snapshot.active && + row.suspendedAt && + row.suspendedAt < purgeCutoff + ) { + deleteMemberIds.push(row.id) + continue + } + const tokenChanged = row.subjectToken !== snapshot.subjectToken + const statusChanged = row.status !== status + if (!tokenChanged && !statusChanged) continue + updates.push({ + id: row.id, + values: { + subjectToken: snapshot.subjectToken, + status, + suspendedAt: snapshot.active ? null : (row.suspendedAt ?? now), + /** A reactivated member is due immediately; their observations may be stale. */ + ...(statusChanged && snapshot.active ? { nextAttemptAt: now, consecutiveFailures: 0 } : {}), + updatedAt: now, + }, + }) + affectedMemberIds.push(row.id) + } + + for (const row of existing) { + if (!snapshots.has(row.credentialId)) deleteMemberIds.push(row.id) + } + + const affectedDocumentIds = new Set() + if (deleteMemberIds.length > 0 || affectedMemberIds.length > 0) { + for (const documentId of await listObservedDocumentIds(db, [ + ...deleteMemberIds, + ...affectedMemberIds, + ])) { + affectedDocumentIds.add(documentId) + } + } + if (updates.length > 0 || deleteMemberIds.length > 0 || inserts.length > 0) { + await withMemberLease(run, async (tx) => { + for (const update of updates) { + await tx + .update(knowledgeConnectorMember) + .set(update.values) + .where(eq(knowledgeConnectorMember.id, update.id)) + } + if (deleteMemberIds.length > 0) { + await tx + .delete(knowledgeConnectorMember) + .where( + and( + eq(knowledgeConnectorMember.connectorId, run.connectorId), + inArray(knowledgeConnectorMember.id, deleteMemberIds) + ) + ) + } + if (inserts.length > 0) { + await tx.insert(knowledgeConnectorMember).values(inserts).onConflictDoNothing() + } + }) + } + + logger.info('Reconciled members-mode membership', { + connectorId: run.connectorId, + credentials: snapshots.size, + inserted: inserts.length, + changed: affectedMemberIds.length, + removed: deleteMemberIds.length, + groupActive: optionState.groupActive, + optionActive: optionState.optionActive, + }) + return { affectedDocumentIds } +} + +/** + * Claims the next due member for this run. Sequential by design: one member + * at a time keeps first-writer-wins deterministic and lets a single huge + * member be aborted at the deadline without touching the others. + */ +async function claimNextMember(run: MemberSyncRun): Promise { + /** + * Proved under the lease: a run reclaimed while it slept must not stamp + * `lastStartedAt`, which would hide the member from its replacement's + * selection and defer that member's access updates to a later run. + */ + const [claimed] = await db.transaction(async (tx) => { + await assertSyncLeaseHeldInTx(tx, run.connectorId, run.lease) + return tx + .update(knowledgeConnectorMember) + .set({ lastStartedAt: new Date(), updatedAt: new Date() }) + .where( + and( + eq(knowledgeConnectorMember.connectorId, run.connectorId), + eq( + knowledgeConnectorMember.id, + sql`( + SELECT ${knowledgeConnectorMember.id} FROM ${knowledgeConnectorMember} + WHERE ${knowledgeConnectorMember.connectorId} = ${run.connectorId} + AND ${knowledgeConnectorMember.status} = 'active' + AND (${knowledgeConnectorMember.nextAttemptAt} IS NULL OR ${knowledgeConnectorMember.nextAttemptAt} <= now()) + AND (${knowledgeConnectorMember.lastStartedAt} IS NULL OR ${knowledgeConnectorMember.lastStartedAt} < ${sql.param(run.runStartedAt, knowledgeConnectorMember.lastStartedAt)}) + ORDER BY ${knowledgeConnectorMember.nextAttemptAt} ASC NULLS FIRST, ${knowledgeConnectorMember.lastStartedAt} ASC NULLS FIRST + LIMIT 1 + FOR UPDATE SKIP LOCKED + )` + ) + ) + ) + .returning() + }) + return claimed ?? null +} + +/** + * Members still due once this run ends, which is what re-dispatch waits for. + * Deliberately ignores `lastStartedAt`: a member this run claimed but could not + * finish is re-armed for now, and the immediate re-dispatch this count + * triggers is what lets them finish. A NULL `nextAttemptAt` means "with the + * connector's next run" — a member that completed on a manual-only connector + * — and must not keep the connector re-dispatching itself. + */ +async function countDueMembers( + run: MemberSyncRun, + binding: { credentialGroupOptionId: string } +): Promise { + const [due] = await db + .select({ count: sql`count(*)::int` }) + .from(knowledgeConnectorMember) + .where( + and( + eq(knowledgeConnectorMember.connectorId, run.connectorId), + eq(knowledgeConnectorMember.status, 'active'), + lte(knowledgeConnectorMember.nextAttemptAt, new Date()) + ) + ) + /** An account that connected while this run was listing has no member row yet. */ + const [unenrolled] = await db + .select({ count: sql`count(*)::int` }) + .from(credential) + .where( + and( + eq(credential.workspaceId, run.workspaceId), + eq(credential.credentialGroupOptionId, binding.credentialGroupOptionId), + eq(credential.type, 'managed_oauth'), + eq(credential.managedOauthStatus, 'active'), + notExists( + db + .select({ one: sql`1` }) + .from(knowledgeConnectorMember) + .where( + and( + eq(knowledgeConnectorMember.connectorId, run.connectorId), + eq(knowledgeConnectorMember.credentialId, credential.id) + ) + ) + ) + ) + ) + return (due?.count ?? 0) + (unenrolled?.count ?? 0) +} + +async function recordMemberFailure( + run: MemberSyncRun, + member: MemberRow, + error: unknown, + syncIntervalMinutes: number +): Promise { + const failures = member.consecutiveFailures + 1 + await withMemberLease(run, (tx) => + tx + .update(knowledgeConnectorMember) + .set({ + consecutiveFailures: failures, + nextAttemptAt: new Date(Date.now() + memberFailureBackoffMs(failures, syncIntervalMinutes)), + lastError: getErrorMessage(error), + updatedAt: new Date(), + }) + .where(eq(knowledgeConnectorMember.id, member.id)) + ) +} + +/** + * Whether a listing failure means the member simply cannot reach the + * configured scope, which is a complete listing of nothing rather than an + * error: the folder or space is not shared with them. + */ +function isScopeUnavailableError(connectorConfig: ConnectorConfig, error: unknown): boolean { + return connectorConfig.isListingScopeUnavailableError?.(error) === true +} + +interface MemberListing { + kind: 'listed' + mode: MemberListingMode + documents: ExternalDocument[] + removedExternalIds: string[] + complete: boolean + /** See {@link MemberListingOutcome.resumable}. */ + resumable: boolean + /** The source itself said this member reaches nothing; not a listing shape to doubt. */ + authoritative: boolean + startedAt: Date + /** Cursor to store once the listing lands: a value, null to close the feed, undefined to leave it. */ + changeCursor: string | null | undefined +} + +async function listForMember(input: { + run: MemberSyncRun + member: MemberRow + connectorConfig: ConnectorConfig + sourceConfig: Record + tokens: MemberTokenCache + syncContext: Record + syncIntervalMinutes: number + /** Relist fully even inside the recrawl window: the member's change feed could not be read. */ + forceFull?: boolean +}): Promise { + const { run, member, connectorConfig, sourceConfig, syncContext } = input + const startedAt = new Date() + const feed = supportsChangeFeed(connectorConfig) + const feedOpen = feed && Boolean(member.changeCursor) + const full = + input.forceFull === true || + shouldListFully( + member.memberSyncedThrough, + member.lastCompleteListingAt, + startedAt, + feedOpen ? MEMBER_CHANGE_FEED_FULL_RECRAWL_MINUTES : MEMBER_FULL_RECRAWL_MINUTES + ) + + try { + if (!full && feed && member.changeCursor) { + let pass: Awaited> + try { + pass = await runChangeFeedPass({ + connectorId: run.connectorId, + connectorConfig, + sourceConfig, + syncContext, + cursor: member.changeCursor, + beforePage: run.lease.beatIfDue, + getAccessToken: () => input.tokens.get(member.id), + deadlineAt: run.deadlineAt, + maxPages: MEMBER_SYNC_MAX_PAGES_PER_MEMBER, + }) + } catch (error) { + if (connectorConfig.isChangeCursorInvalidError?.(error) !== true) throw error + logger.warn('Member change feed cursor rejected; reopening it from a full listing', { + connectorId: run.connectorId, + memberId: member.id, + error: getErrorMessage(error), + }) + return listForMember({ ...input, forceFull: true }) + } + const complete = pass.exhausted && !pass.budgetAborted + return { + kind: 'listed', + mode: 'changes', + documents: pass.upserts, + removedExternalIds: pass.removedExternalIds, + complete, + /** The cursor already sits past every page read, so the next run continues. */ + resumable: !complete, + authoritative: false, + startedAt, + changeCursor: pass.cursor, + } + } + + /** + * The feed opens before the listing starts so a change that lands while + * the listing is running is reported by the first feed read instead of + * waiting for the next full listing. + */ + const openedCursor = + full && feed + ? await connectorConfig.getChangeCursor( + await input.tokens.get(member.id), + sourceConfig, + syncContext + ) + : undefined + const lastSyncAt = + full || !member.memberSyncedThrough + ? undefined + : new Date(member.memberSyncedThrough.getTime() - INCREMENTAL_OVERLAP_MS) + /** + * A listing pass has no cursor to resume from, so a page cap would relist + * the same first pages every run and never reach the documents behind + * them. The run's deadline is its only bound: a member the budget cuts off + * is re-armed at once (`resumable`), and one no run can finish alone + * backs off (`exhaustedRunAlone`) instead of being silently truncated. + */ + const listing = await runListingPass({ + connectorId: run.connectorId, + connectorConfig, + sourceConfig, + syncContext, + lastSyncAt, + beforePage: run.lease.beatIfDue, + getAccessToken: () => input.tokens.get(member.id), + deadlineAt: run.deadlineAt, + maxPages: Number.POSITIVE_INFINITY, + }) + const complete = + listing.exhausted && + !listing.budgetAborted && + !syncContext.listingCapped && + !syncContext.reconciliationUnsafe + return { + kind: 'listed', + mode: full ? 'full' : 'incremental', + documents: listing.documents, + removedExternalIds: [], + complete, + /** Only the deadline is worth retrying at once; a capped or truncated source reads the same next time. */ + resumable: listing.budgetAborted, + authoritative: false, + startedAt, + changeCursor: full && complete ? openedCursor : undefined, + } + } catch (error) { + if (error instanceof SyncLockLostException || error instanceof ConnectorSyncCapacityError) { + throw error + } + if (isRateLimitError(error)) throw error + if (isScopeUnavailableError(connectorConfig, error)) { + logger.info('Member cannot reach the configured source scope; treating as an empty listing', { + connectorId: run.connectorId, + memberId: member.id, + }) + return { + kind: 'listed', + mode: 'full', + documents: [], + removedExternalIds: [], + complete: true, + resumable: false, + authoritative: true, + startedAt, + /** A feed over a scope the member cannot reach says nothing; the next full listing reopens one. */ + changeCursor: null, + } + } + logger.warn('Member listing failed', { + connectorId: run.connectorId, + memberId: member.id, + error: getErrorMessage(error), + }) + await recordMemberFailure(input.run, member, error, input.syncIntervalMinutes) + run.result.membersFailed += 1 + return { kind: 'failed' } + } +} + +/** + * Writes what one member's listing established: observations for everything + * they saw, removals only after a full, complete, non-suspect listing or by + * the change feed's explicit word, and the member's schedule, watermark, and + * feed cursor. Returns the documents whose ACL changed. + */ +async function applyMemberListing( + run: MemberSyncRun, + outcome: MemberListingOutcome, + documentIdByExternalId: Map, + syncIntervalMinutes: number +): Promise> { + const affected = new Set() + const seenDocumentIds: string[] = [] + for (const externalId of outcome.seenExternalIds) { + const documentId = documentIdByExternalId.get(externalId) + if (documentId) seenDocumentIds.push(documentId) + } + const removesAllowed = outcome.mode === 'full' && outcome.complete && !outcome.suspect + const exhaustedFailures = (outcome.member.consecutiveFailures ?? 0) + 1 + const now = new Date() + + await withMemberLease(run, async (tx) => { + const added = await recordMemberObservations(tx, outcome.member.id, seenDocumentIds, run.runId) + run.result.observationsAdded += added + /** + * Every seen document is rematerialised, not only the newly observed ones: + * a run that died between writing observations and writing ACLs left them + * hidden, and the observation graph is the only record that says so. + * Rematerialising an already-correct ACL is a no-op write. + */ + for (const documentId of seenDocumentIds) affected.add(documentId) + if (removesAllowed) { + const removed = await removeUnseenMemberObservations(tx, outcome.member.id, run.runId) + run.result.observationsRemoved += removed.length + for (const documentId of removed) affected.add(documentId) + } else if (outcome.mode === 'changes') { + const removedDocumentIds: string[] = [] + for (const externalId of outcome.removedExternalIds) { + const documentId = documentIdByExternalId.get(externalId) + if (documentId) removedDocumentIds.push(documentId) + } + const removed = await removeMemberObservationsForDocuments( + tx, + outcome.member.id, + removedDocumentIds + ) + run.result.observationsRemoved += removed.length + for (const documentId of removed) affected.add(documentId) + } + await tx + .update(knowledgeConnectorMember) + .set({ + ...(outcome.exhaustedRunAlone + ? { + consecutiveFailures: exhaustedFailures, + lastError: 'Listing did not finish within one run', + } + : { consecutiveFailures: 0, lastError: null }), + ...(outcome.mode === 'full' ? { lastListedCount: outcome.listedCount } : {}), + nextAttemptAt: outcome.exhaustedRunAlone + ? new Date(now.getTime() + memberFailureBackoffMs(exhaustedFailures, syncIntervalMinutes)) + : outcome.resumable + ? now + : memberNextAttemptAt(now, syncIntervalMinutes), + ...(removesAllowed + ? { lastCompleteListingAt: now, memberSyncedThrough: outcome.listingStartedAt } + : {}), + ...(outcome.mode === 'changes' && outcome.complete + ? { memberSyncedThrough: outcome.listingStartedAt } + : {}), + ...(outcome.changeCursor !== undefined ? { changeCursor: outcome.changeCursor } : {}), + updatedAt: now, + }) + .where(eq(knowledgeConnectorMember.id, outcome.member.id)) + }) + + if (outcome.complete) run.result.membersCompleted += 1 + else run.result.membersIncomplete += 1 + return affected +} + +async function loadDocumentIdsByExternalId(connectorId: string): Promise> { + const rows = await db + .select({ id: document.id, externalId: document.externalId }) + .from(document) + .where(and(eq(document.connectorId, connectorId), isNull(document.archivedAt))) + .limit(syncWorkingSetQueryLimit(0)) + const byExternalId = new Map() + for (const row of rows) { + if (row.externalId && !byExternalId.has(row.externalId)) + byExternalId.set(row.externalId, row.id) + } + return byExternalId +} + +async function completeMemberSync( + run: MemberSyncRun, + syncIntervalMinutes: number +): Promise { + const { result } = run + const now = new Date() + const nextMemberSyncAt = nextMemberSyncTime(now, syncIntervalMinutes, result.membersRemaining) + return db.transaction(async (tx) => { + const [activeKnowledgeBase] = await tx + .select({ id: knowledgeBase.id }) + .from(knowledgeBase) + .where(and(eq(knowledgeBase.id, run.knowledgeBaseId), isNull(knowledgeBase.deletedAt))) + .for('update') + if (!activeKnowledgeBase) { + /** Nothing to record against a deleted knowledge base; hand the lease back rather than let it expire as a failure. */ + await tx + .update(knowledgeConnectorMemberSyncLog) + .set({ + status: 'failed', + completedAt: now, + errorMessage: 'Knowledge base deleted during sync', + }) + .where( + and( + eq(knowledgeConnectorMemberSyncLog.id, run.runId), + eq(knowledgeConnectorMemberSyncLog.status, 'started') + ) + ) + await tx + .update(knowledgeConnector) + .set({ + memberSyncStatus: 'idle', + nextMemberSyncAt: null, + memberSyncLockToken: null, + memberSyncLockLeaseAt: null, + updatedAt: now, + }) + .where(stillHoldsMemberSyncLock(run.connectorId, run.runId)) + return false + } + const [held] = await tx + .select({ id: knowledgeConnector.id }) + .from(knowledgeConnector) + .where(stillHoldsMemberSyncLock(run.connectorId, run.runId)) + .for('update') + if (!held) return false + + const [closedLog] = await tx + .update(knowledgeConnectorMemberSyncLog) + .set({ + status: 'completed', + completedAt: now, + membersClaimed: result.membersClaimed, + membersCompleted: result.membersCompleted, + membersIncomplete: result.membersIncomplete, + membersFailed: result.membersFailed, + docsListed: result.docsListed, + docsAdded: result.docsAdded, + docsUpdated: result.docsUpdated, + docsUnchanged: result.docsUnchanged, + docsHydratedOnce: result.docsHydratedOnce, + observationsAdded: result.observationsAdded, + observationsRemoved: result.observationsRemoved, + docsTombstoned: result.docsTombstoned, + docsResurrected: result.docsResurrected, + docsPurged: result.docsPurged, + credentialsAudited: result.credentialsAudited, + }) + .where( + and( + eq(knowledgeConnectorMemberSyncLog.id, run.runId), + eq(knowledgeConnectorMemberSyncLog.status, 'started') + ) + ) + .returning({ id: knowledgeConnectorMemberSyncLog.id }) + if (!closedLog) return false + + const [written] = await tx + .update(knowledgeConnector) + .set({ + memberSyncStatus: 'idle', + lastMemberSyncAt: now, + nextMemberSyncAt, + lastMemberSyncError: null, + memberSyncConsecutiveFailures: 0, + memberSyncLockToken: null, + memberSyncLockLeaseAt: null, + updatedAt: now, + }) + .where(stillHoldsMemberSyncLock(run.connectorId, run.runId)) + .returning({ id: knowledgeConnector.id }) + return Boolean(written) + }) +} + +async function failMemberSyncLog(runId: string, result: MemberSyncResult, errorMessage: string) { + await db + .update(knowledgeConnectorMemberSyncLog) + .set({ + status: 'failed', + completedAt: new Date(), + errorMessage, + membersClaimed: result.membersClaimed, + membersCompleted: result.membersCompleted, + membersIncomplete: result.membersIncomplete, + membersFailed: result.membersFailed, + docsListed: result.docsListed, + docsAdded: result.docsAdded, + docsUpdated: result.docsUpdated, + docsUnchanged: result.docsUnchanged, + docsHydratedOnce: result.docsHydratedOnce, + observationsAdded: result.observationsAdded, + observationsRemoved: result.observationsRemoved, + docsTombstoned: result.docsTombstoned, + docsResurrected: result.docsResurrected, + docsPurged: result.docsPurged, + credentialsAudited: result.credentialsAudited, + }) + .where( + and( + eq(knowledgeConnectorMemberSyncLog.id, runId), + eq(knowledgeConnectorMemberSyncLog.status, 'started') + ) + ) +} + +/** + * Ends a run without doing anything because the feature is not available to + * the workspace right now. The connector keeps its members and their + * observations, and its failure ladder does not advance; the reason is left + * on the connector and the run's log so an admin can see why nothing syncs. + * It is looked at again on its next schedule; a manual-only connector waits + * for the next manual sync. + */ +async function deferMemberSync(run: MemberSyncRun, syncIntervalMinutes: number): Promise { + const now = new Date() + await failMemberSyncLog(run.runId, run.result, 'Per-member access is not available; waiting') + await db + .update(knowledgeConnector) + .set({ + memberSyncStatus: 'idle', + lastMemberSyncError: 'Per-member access is not available for this workspace', + nextMemberSyncAt: nextMemberSyncTime(now, syncIntervalMinutes, false), + memberSyncLockToken: null, + memberSyncLockLeaseAt: null, + updatedAt: now, + }) + .where(holdsMemberSyncLockToken(run.connectorId, run.runId)) + logger.info('Member sync deferred; per-member access is not available', { + connectorId: run.connectorId, + }) +} + +/** + * Disables member sync on a connector that can no longer run it because its + * group binding is gone, and suspends every member so their tokens leave + * every ACL. Nothing is purged: re-enabling restores access from the retained + * observations. + */ +async function disableMemberSync(run: MemberSyncRun, reason: string): Promise { + const now = new Date() + /** Suspension, the ACLs it changes, and the disable itself land together, and only under the lease. */ + await withMemberLease(run, async (tx) => { + const suspended = await tx + .update(knowledgeConnectorMember) + .set({ status: 'suspended', suspendedAt: now, updatedAt: now }) + .where( + and( + eq(knowledgeConnectorMember.connectorId, run.connectorId), + eq(knowledgeConnectorMember.status, 'active') + ) + ) + .returning({ id: knowledgeConnectorMember.id }) + if (suspended.length > 0) { + const affected = await listObservedDocumentIds( + tx, + suspended.map((row) => row.id) + ) + await materializeDocumentAcls(run.connectorId, affected, tx) + } + await tx + .update(knowledgeConnector) + .set({ + memberSyncStatus: 'disabled', + lastMemberSyncError: reason, + nextMemberSyncAt: null, + memberSyncLockToken: null, + memberSyncLockLeaseAt: null, + updatedAt: now, + }) + .where(holdsMemberSyncLockToken(run.connectorId, run.runId)) + }) + await failMemberSyncLog(run.runId, run.result, reason) + logger.warn('Member sync disabled', { connectorId: run.connectorId, reason }) +} + +/** + * Executes one members-mode run for a connector: reconciles membership from + * the credential-group option, crawls the source once per due member with that + * member's own token until the budget ends, hydrates every listed document + * once, records who observed what, materialises the ACLs, applies the document + * lifecycle, and re-dispatches itself while members remain due. + */ +export async function executeMemberSync( + connectorId: string, + options: ExecuteMemberSyncOptions +): Promise { + const billingAttribution = assertBillingAttributionSnapshot(options.billingAttribution) + const result = emptyResult() + + const [connectorBeforeLock] = await db + .select() + .from(knowledgeConnector) + .where( + and( + eq(knowledgeConnector.id, connectorId), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + .limit(1) + if (!connectorBeforeLock) { + logger.warn('Skipping member sync: connector not found, archived, or deleted', { connectorId }) + return skipped(result, 'connector_unavailable') + } + if (connectorBeforeLock.accessMode !== 'members') { + logger.info('Skipping member sync: connector does not sync per member', { connectorId }) + return skipped(result, 'connector_not_syncable') + } + const connectorConfig = CONNECTOR_REGISTRY[connectorBeforeLock.connectorType] + if (!connectorConfig) { + throw new Error(`Unknown connector type: ${connectorBeforeLock.connectorType}`) + } + + const [kbRow] = await db + .select({ userId: knowledgeBase.userId, workspaceId: knowledgeBase.workspaceId }) + .from(knowledgeBase) + .where( + and( + eq(knowledgeBase.id, connectorBeforeLock.knowledgeBaseId), + isNull(knowledgeBase.deletedAt) + ) + ) + .limit(1) + if (!kbRow) { + logger.warn('Skipping member sync: knowledge base is deleted', { connectorId }) + await db + .update(knowledgeConnector) + .set({ + memberSyncStatus: 'error', + nextMemberSyncAt: null, + lastMemberSyncError: 'Knowledge base deleted', + memberSyncLockToken: null, + memberSyncLockLeaseAt: null, + updatedAt: new Date(), + }) + .where(eq(knowledgeConnector.id, connectorId)) + return skipped(result, 'knowledge_base_deleted') + } + if (!kbRow.workspaceId) { + throw new Error( + `Knowledge base ${connectorBeforeLock.knowledgeBaseId} is missing workspace billing context` + ) + } + if (billingAttribution.workspaceId !== kbRow.workspaceId) { + throw new Error( + `Member sync billing attribution does not match knowledge base workspace ${kbRow.workspaceId}` + ) + } + const kbOwner: KnowledgeBaseOwner = { workspaceId: kbRow.workspaceId, userId: kbRow.userId } + + const runId = generateId() + const connector = await acquireMemberSyncLock(connectorId, runId, options.dispatchToken) + if (!connector) { + const [current] = await db + .select({ + status: knowledgeConnector.status, + memberSyncStatus: knowledgeConnector.memberSyncStatus, + memberSyncLockToken: knowledgeConnector.memberSyncLockToken, + syncLockToken: knowledgeConnector.syncLockToken, + }) + .from(knowledgeConnector) + .where(eq(knowledgeConnector.id, connectorId)) + .limit(1) + if ( + current?.memberSyncStatus === 'disabled' || + current?.syncLockToken || + (current && !MEMBER_LOCKABLE_CONNECTOR_STATUSES.some((status) => status === current.status)) + ) { + logger.info('Connector is not accepting member syncs, skipping', { + connectorId, + status: current.status, + }) + return skipped(result, 'connector_not_syncable') + } + if (options.dispatchToken && current?.memberSyncLockToken !== options.dispatchToken) { + logger.info('Member sync superseded by a newer dispatch, skipping', { connectorId }) + return skipped(result, 'dispatch_superseded') + } + logger.info('Member sync already in progress, skipping', { connectorId }) + return skipped(result, 'sync_in_progress') + } + + const runStartedAt = new Date() + const run: MemberSyncRun = { + connectorId, + knowledgeBaseId: connector.knowledgeBaseId, + workspaceId: kbRow.workspaceId, + runId, + runStartedAt, + deadlineAt: runStartedAt.getTime() + MEMBER_SYNC_SOFT_BUDGET_SECONDS * 1000, + result, + lease: createMemberSyncLease(connectorId, runId), + } + await insertMemberSyncLog(runId, connectorId, runStartedAt) + + try { + /** + * Where the feature is off — flag, plan, or a flag read that could not + * reach its source — nothing changes: readers already see no member-scoped + * document, and the run waits for the next schedule to look again. + */ + if (!(await isKnowledgeMemberAccessAvailable({ workspaceId: run.workspaceId }))) { + await deferMemberSync(run, connector.syncIntervalMinutes) + return { + ...skipped(result, 'connector_not_syncable'), + error: 'Per-member access is not available for this workspace', + } + } + if (!connector.credentialGroupId || !connector.credentialGroupOptionId) { + await disableMemberSync(run, 'Connector is no longer attached to a Credential Group option') + return { + ...skipped(result, 'connector_not_syncable'), + error: 'Connector is no longer attached to a Credential Group option', + } + } + if (!connectorConfig.permissionScopedListing || connectorConfig.auth.mode !== 'oauth') { + throw new Error(`Connector ${connectorConfig.id} cannot sync per member`) + } + const binding = { + credentialGroupId: connector.credentialGroupId, + credentialGroupOptionId: connector.credentialGroupOptionId, + } + const sourceConfig = connector.sourceConfig as Record + + if (connector.accessRewritePending && !(await finishPendingAccessRewrite(run))) { + /** The rewrite is not done, so nothing is listed yet; the next run picks it up at once. */ + result.membersRemaining = true + const landed = await completeMemberSync(run, connector.syncIntervalMinutes) + if (!landed) return skipped(result, 'sync_superseded') + logger.info('Member sync spent its budget hiding documents after a mode switch', { + connectorId, + runId, + }) + return result + } + + const affectedDocumentIds = new Set() + /** + * Anyone who joined the workspace since the last run is invited now, so + * membership grows on its own; the invitation is the only thing they need. + */ + const invited = await inviteWorkspaceMembersToCredentialGroup({ + workspaceId: run.workspaceId, + credentialGroupId: connector.credentialGroupId, + beforeBatch: run.lease.beatIfDue, + }).catch((error) => { + logger.warn('Failed to invite new workspace members during a member run', { + connectorId, + error: getErrorMessage(error), + }) + return null + }) + if (invited && invited.invited > 0) { + logger.info('Invited new workspace members to the connector credential group', { + connectorId, + ...invited, + }) + } + const membership = await reconcileMembership(run, binding) + for (const documentId of membership.affectedDocumentIds) affectedDocumentIds.add(documentId) + + const members = await db + .select({ + id: knowledgeConnectorMember.id, + credentialId: knowledgeConnectorMember.credentialId, + }) + .from(knowledgeConnectorMember) + .where( + and( + eq(knowledgeConnectorMember.connectorId, connectorId), + eq(knowledgeConnectorMember.status, 'active') + ) + ) + const credentialIdByMemberId = new Map( + members.map((member) => [member.id, member.credentialId]) + ) + const tokens = createMemberTokenCache({ run, connectorConfig, credentialIdByMemberId }) + const syncContexts = new Map>() + + const union = new Map() + const outcomes: MemberListingOutcome[] = [] + let retainedBytes = 0 + + while (Date.now() < run.deadlineAt) { + const member = await claimNextMember(run) + if (!member) break + result.membersClaimed += 1 + const syncContext: Record = { + syncRunId: runId, + memberId: member.id, + ...PER_MEMBER_LISTING_CONTEXT, + } + syncContexts.set(member.id, syncContext) + + const listed = await listForMember({ + run, + member, + connectorConfig, + sourceConfig, + tokens, + syncContext, + syncIntervalMinutes: connector.syncIntervalMinutes, + }) + if (listed.kind === 'failed') continue + + const admitted = admitMemberListing( + union, + member.id, + listed.documents, + connectorId, + retainedBytes + ) + retainedBytes = admitted.retainedBytes + result.docsListed += admitted.seenExternalIds.size + /** + * A listing that collapsed against the member's previous one is doubted + * once: removals wait for the next full listing to say the same, which it + * does by then comparing against the collapsed count. A source that said + * outright the member reaches nothing is not a shape to doubt. + */ + const suspect = + listed.mode === 'full' && + !listed.authoritative && + classifySuspectListing(admitted.seenExternalIds.size, member.lastListedCount ?? 0) !== null + if (suspect) { + logger.warn('Suspect member listing; removals withheld', { + connectorId, + memberId: member.id, + listed: admitted.seenExternalIds.size, + previouslyListed: member.lastListedCount, + }) + } + outcomes.push({ + member, + mode: listed.mode, + listingStartedAt: listed.startedAt, + seenExternalIds: admitted.seenExternalIds, + removedExternalIds: listed.removedExternalIds, + listedCount: admitted.seenExternalIds.size, + complete: listed.complete, + resumable: listed.resumable, + exhaustedRunAlone: + listed.resumable && result.membersClaimed === 1 && Date.now() >= run.deadlineAt, + suspect, + /** A doubted listing does not open the feed either: the next full listing decides. */ + changeCursor: suspect ? undefined : listed.changeCursor, + }) + } + + const corpus = await loadOwnedCorpus(connectorId) + const state = createSyncRunState(result) + const externalDocs = [...union.values()].map((entry) => entry.document) + const pendingOps = classifyListing({ externalDocs, corpus, forceRehydrate: false, state }) + result.docsHydratedOnce = pendingOps.filter( + (op) => op.type !== 'skip' && op.extDoc.contentDeferred + ).length + + await processDocOps({ + connectorId, + connector, + sourceConfig, + kbOwner, + billingAttribution, + pendingOps, + corpus, + forceRehydrate: false, + state, + hydration: { + getDocument: async (externalId) => { + const observers = union.get(externalId)?.observers ?? [] + let lastError: unknown + for (const memberId of observers.slice(0, HYDRATION_OBSERVER_ATTEMPTS)) { + try { + const accessToken = await tokens.get(memberId) + const hydrated = await connectorConfig.getDocument( + accessToken, + sourceConfig, + externalId, + syncContexts.get(memberId) + ) + if (hydrated) return hydrated + } catch (error) { + if (isRateLimitError(error)) throw error + if (error instanceof KnowledgeConnectorMemberAccessDeniedError) continue + lastError = error + } + } + if (lastError) throw lastError + return null + }, + }, + lease: run.lease, + documentAccess: 'members', + }) + + const documentIdByExternalId = await loadDocumentIdsByExternalId(connectorId) + for (const outcome of outcomes) { + await run.lease.beatIfDue() + const affected = await applyMemberListing( + run, + outcome, + documentIdByExternalId, + connector.syncIntervalMinutes + ) + for (const documentId of affected) affectedDocumentIds.add(documentId) + } + + await withMemberLease(run, (tx) => + materializeDocumentAcls(connectorId, affectedDocumentIds, tx) + ) + + /** + * Nobody has completed a listing yet — a connector that just entered + * members mode, waiting for its first member to connect — so an + * unobserved document says nothing about access and must not be + * tombstoned, let alone purged a week later. + */ + const [listed] = await db + .select({ count: sql`count(*)::int` }) + .from(knowledgeConnectorMember) + .where( + and( + eq(knowledgeConnectorMember.connectorId, connectorId), + sql`${knowledgeConnectorMember.lastCompleteListingAt} IS NOT NULL` + ) + ) + const lifecycle = await applyMemberDocumentLifecycle({ + connectorId, + knowledgeBaseId: connector.knowledgeBaseId, + runId, + lease: run.lease, + withLease: (fn) => withMemberLease(run, fn), + failedExternalIds: state.failedExternalIds, + allowRemoval: (listed?.count ?? 0) > 0, + }) + result.docsTombstoned = lifecycle.tombstoned + result.docsResurrected = lifecycle.resurrected + result.docsPurged = lifecycle.purged + result.docsDeleted = lifecycle.purged + + await sweepStuckDocuments({ + connectorId, + knowledgeBaseId: connector.knowledgeBaseId, + syncStartedAt: runStartedAt, + retryCutoff: new Date(Date.now() - RETRY_WINDOW_DAYS * 24 * 60 * 60 * 1000), + billingAttribution, + result, + lease: run.lease, + }) + + result.membersRemaining = (await countDueMembers(run, binding)) > 0 + const landed = await completeMemberSync(run, connector.syncIntervalMinutes) + if (!landed) { + logger.warn( + 'Member sync result discarded — connector was reclaimed while this run was executing', + { + connectorId, + runId, + } + ) + return skipped(result, 'sync_superseded') + } + logger.info('Member sync completed', { connectorId, runId, ...result }) + return result + } catch (error) { + if (error instanceof SyncLockLostException) { + logger.warn('Member sync abandoned — lock was reclaimed while this run was executing', { + connectorId, + runId, + }) + return skipped(result, 'sync_superseded') + } + if (error instanceof ConnectorDeletedException) { + logger.info('Connector deleted during member sync', { connectorId }) + await failMemberSyncLog(runId, result, 'Connector deleted during sync').catch((logError) => + logger.error('Failed to record member sync failure', { + connectorId, + error: getErrorMessage(logError), + }) + ) + return skipped(result, 'connector_deleted_during_sync') + } + if (error instanceof MemberBindingGoneError) { + try { + await disableMemberSync(run, error.message) + } catch (disableError) { + if (!(disableError instanceof SyncLockLostException)) throw disableError + logger.warn('Member sync abandoned — lock was reclaimed before it could be disabled', { + connectorId, + runId, + }) + return skipped(result, 'sync_superseded') + } + return { ...skipped(result, 'connector_not_syncable'), error: error.message } + } + + const errorMessage = toError(error).message + const retryAfterMs = getRetryAfterMs(error) + logger.error('Member sync failed', { connectorId, runId, error: errorMessage }) + try { + await failMemberSyncLog(runId, result, errorMessage) + const failureUpdate = + error instanceof ConnectorSyncCapacityError + ? { + memberSyncStatus: 'error' as const, + lastMemberSyncError: errorMessage, + nextMemberSyncAt: null, + memberSyncConsecutiveFailures: connector.memberSyncConsecutiveFailures, + memberSyncLockToken: null, + memberSyncLockLeaseAt: null, + updatedAt: new Date(), + } + : buildMemberSyncFailureUpdate( + new Date(), + connector.memberSyncConsecutiveFailures, + errorMessage, + retryAfterMs + ) + const written = await db + .update(knowledgeConnector) + .set(failureUpdate) + .where(stillHoldsMemberSyncLock(connectorId, runId)) + .returning({ id: knowledgeConnector.id }) + if (written.length === 0) { + logger.warn('Member sync failure discarded — connector was reclaimed', { + connectorId, + runId, + }) + } + } catch (recoveryError) { + logger.error('Failed to record member sync failure', { + connectorId, + error: toError(recoveryError).message, + }) + } + result.error = errorMessage + return result + } +} diff --git a/apps/sim/lib/knowledge/connectors/queue.test.ts b/apps/sim/lib/knowledge/connectors/queue.test.ts index 6c5cbc70dd7..f7e7d0ec807 100644 --- a/apps/sim/lib/knowledge/connectors/queue.test.ts +++ b/apps/sim/lib/knowledge/connectors/queue.test.ts @@ -39,6 +39,9 @@ vi.mock('@/lib/knowledge/documents/service', () => ({ vi.mock('@/lib/knowledge/connectors/sync-engine', () => ({ executeSync: mockExecuteSync, isConnectorRunnableStatus: (status: string) => status === 'active' || status === 'error', +})) + +vi.mock('@/lib/knowledge/connectors/sync-lock', () => ({ connectorIsLive: () => ({ type: 'connectorIsLive' }), LOCKABLE_CONNECTOR_STATUSES: ['active', 'error', 'pending'], })) @@ -79,6 +82,7 @@ describe('connector sync queue', () => { { knowledgeBaseId: 'knowledge-base-1', connectorStatus: 'active', + connectorAccessMode: 'workspace', connectorArchivedAt: null, connectorDeletedAt: null, connectorNextSyncAt: NEXT_SYNC_AT, @@ -194,6 +198,7 @@ describe('connector sync queue', () => { { knowledgeBaseId: 'knowledge-base-1', connectorStatus: 'paused', + connectorAccessMode: 'workspace', connectorArchivedAt: null, connectorDeletedAt: null, workspaceId: 'workspace-paid', @@ -232,6 +237,7 @@ describe('connector sync queue', () => { { knowledgeBaseId: 'knowledge-base-1', connectorStatus: 'paused', + connectorAccessMode: 'workspace', connectorArchivedAt: null, connectorDeletedAt: null, connectorNextSyncAt: NEXT_SYNC_AT, diff --git a/apps/sim/lib/knowledge/connectors/queue.ts b/apps/sim/lib/knowledge/connectors/queue.ts index e1c071fed6f..4d4de0e463c 100644 --- a/apps/sim/lib/knowledge/connectors/queue.ts +++ b/apps/sim/lib/knowledge/connectors/queue.ts @@ -11,12 +11,8 @@ import { type BillingAttributionSnapshot, } from '@/lib/billing/core/billing-attribution' import { resolveTriggerRegion } from '@/lib/core/async-jobs/region' -import { - connectorIsLive, - executeSync, - isConnectorRunnableStatus, - LOCKABLE_CONNECTOR_STATUSES, -} from '@/lib/knowledge/connectors/sync-engine' +import { executeSync, isConnectorRunnableStatus } from '@/lib/knowledge/connectors/sync-engine' +import { connectorIsLive, LOCKABLE_CONNECTOR_STATUSES } from '@/lib/knowledge/connectors/sync-lock' import { isTriggerAvailable } from '@/lib/knowledge/documents/service' const logger = createLogger('ConnectorSyncQueue') @@ -165,6 +161,7 @@ async function markSyncPending(connectorId: string): Promise { .where( and( eq(knowledgeConnector.id, connectorId), + eq(knowledgeConnector.accessMode, 'workspace'), inArray(knowledgeConnector.status, LOCKABLE_CONNECTOR_STATUSES), isNull(knowledgeConnector.syncLockToken), connectorIsLive() @@ -296,6 +293,7 @@ export async function dispatchSync( .select({ knowledgeBaseId: knowledgeConnector.knowledgeBaseId, connectorStatus: knowledgeConnector.status, + connectorAccessMode: knowledgeConnector.accessMode, connectorArchivedAt: knowledgeConnector.archivedAt, connectorDeletedAt: knowledgeConnector.deletedAt, connectorNextSyncAt: knowledgeConnector.nextSyncAt, @@ -349,6 +347,13 @@ export async function dispatchSync( }) return { queued: false, reason: 'Connector has been archived or deleted' } } + if (row.connectorAccessMode !== 'workspace') { + logger.info('Skipping sync dispatch: connector syncs per member', { connectorId, requestId }) + return { + queued: false, + reason: 'Connector syncs per member and is not synced as the workspace', + } + } if (payload.requireRunnable && !isConnectorRunnableStatus(row.connectorStatus)) { logger.info('Skipping automatic sync dispatch: connector is not runnable', { connectorId, diff --git a/apps/sim/lib/knowledge/connectors/source-modified-at.test.ts b/apps/sim/lib/knowledge/connectors/source-modified-at.test.ts new file mode 100644 index 00000000000..492ab74fc31 --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/source-modified-at.test.ts @@ -0,0 +1,74 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { resolveSourceModifiedAt } from '@/lib/knowledge/connectors/source-modified-at' + +const NOW = new Date('2026-09-01T12:00:00Z') + +describe('resolveSourceModifiedAt', () => { + it.each([ + ['modifiedTime', '2026-08-20T12:00:00Z'], + ['updatedTime', '2026-08-20T12:00:00Z'], + ['lastModified', '2026-08-20T12:00:00.000Z'], + ['lastModifiedDateTime', '2026-08-20T12:00:00Z'], + ['updatedAt', '2026-08-20T12:00:00Z'], + ['updated', '2026-08-20T12:00:00Z'], + ['lastUpdated', '2026-08-20 12:00:00Z'], + ['statusDate', '2026-08-20T12:00:00.000+0000'], + ])('reads %s', (key, value) => { + expect(resolveSourceModifiedAt({ [key]: value }, NOW)?.toISOString()).toBe( + '2026-08-20T12:00:00.000Z' + ) + }) + + it('accepts Date objects and epoch seconds or milliseconds', () => { + const at = new Date('2026-08-20T12:00:00Z') + expect(resolveSourceModifiedAt({ modifiedAt: at }, NOW)).toBe(at) + expect(resolveSourceModifiedAt({ updatedAt: at.getTime() }, NOW)?.getTime()).toBe(at.getTime()) + expect(resolveSourceModifiedAt({ updatedAt: at.getTime() / 1000 }, NOW)?.getTime()).toBe( + at.getTime() + ) + }) + + it('prefers the earlier key and skips values that are not plausible timestamps', () => { + expect( + resolveSourceModifiedAt( + { updatedAt: '2026-08-01T00:00:00Z', modifiedTime: '2026-08-20T12:00:00Z' }, + NOW + )?.toISOString() + ).toBe('2026-08-20T12:00:00.000Z') + expect( + resolveSourceModifiedAt( + { modifiedTime: 'yesterday', updatedAt: '2026-08-01T00:00:00Z' }, + NOW + )?.toISOString() + ).toBe('2026-08-01T00:00:00.000Z') + }) + + it('rejects an invalid Date instance and a number outside the Date range', () => { + expect(resolveSourceModifiedAt({ modifiedTime: new Date('not a date') })).toBeNull() + expect(resolveSourceModifiedAt({ modifiedTime: 1e20 })).toBeNull() + }) + + it('reads the newest message time an email conversation reports', () => { + expect( + resolveSourceModifiedAt({ lastMessageDate: '2026-08-29T09:30:00Z' })?.toISOString() + ).toBe('2026-08-29T09:30:00.000Z') + }) + + it('reads the last activity a chat space or channel reports', () => { + expect(resolveSourceModifiedAt({ lastActivity: '2026-08-30T10:00:00Z' })?.toISOString()).toBe( + '2026-08-30T10:00:00.000Z' + ) + }) + + it('rejects placeholders and far-future values', () => { + expect(resolveSourceModifiedAt({ modifiedTime: 0 }, NOW)).toBeNull() + expect(resolveSourceModifiedAt({ modifiedTime: '1970-01-01T00:00:00Z' }, NOW)).toBeNull() + expect(resolveSourceModifiedAt({ modifiedTime: '2030-01-01T00:00:00Z' }, NOW)).toBeNull() + expect(resolveSourceModifiedAt({ modifiedTime: '' }, NOW)).toBeNull() + expect(resolveSourceModifiedAt(undefined, NOW)).toBeNull() + expect(resolveSourceModifiedAt({}, NOW)).toBeNull() + }) +}) diff --git a/apps/sim/lib/knowledge/connectors/source-modified-at.ts b/apps/sim/lib/knowledge/connectors/source-modified-at.ts new file mode 100644 index 00000000000..47ecf923231 --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/source-modified-at.ts @@ -0,0 +1,63 @@ +/** + * The metadata keys connectors use for the source's last-modified time, in + * the order they are tried. Connectors were never asked to agree on a name, + * so the persisted column is derived here rather than in each of them. + */ +const SOURCE_MODIFIED_AT_KEYS = [ + 'modifiedTime', + 'updatedTime', + 'lastModified', + 'lastModifiedDateTime', + 'modifiedAt', + 'modified_at', + 'lastUpdated', + 'updatedAt', + 'updated_at', + 'updated', + /** JSM requests: the time the request last changed status, the list endpoint's only change signal. */ + 'statusDate', + /** Google Chat spaces and Teams channels: the latest message time, the listing's only change signal. */ + 'lastActivity', + /** Gmail and Outlook conversations: the newest message's time. */ + 'lastMessageDate', +] as const + +/** Earlier than any plausible document; guards against epoch-zero placeholders. */ +const EARLIEST_PLAUSIBLE_MS = Date.UTC(1990, 0, 1) +/** A source clock a day ahead is skew; further ahead is a placeholder. */ +const FUTURE_TOLERANCE_MS = 24 * 60 * 60 * 1000 + +/** A `Date` only when it holds a real instant; a finite number outside the Date range yields an invalid one. */ +function validDate(date: Date): Date | null { + return Number.isNaN(date.getTime()) ? null : date +} + +function toDate(value: unknown): Date | null { + if (value instanceof Date) return validDate(value) + if (typeof value === 'number' && Number.isFinite(value)) { + /** Seconds-since-epoch values are far too small to be milliseconds after 1990. */ + return validDate(new Date(value < 1e11 ? value * 1000 : value)) + } + if (typeof value === 'string' && value.trim()) return validDate(new Date(value)) + return null +} + +/** + * The source's last-modified time from a connector's document metadata, or + * null when it reports none or the value is not a plausible timestamp. + */ +export function resolveSourceModifiedAt( + metadata: Record | undefined, + now: Date = new Date() +): Date | null { + if (!metadata) return null + for (const key of SOURCE_MODIFIED_AT_KEYS) { + if (!(key in metadata)) continue + const parsed = toDate(metadata[key]) + if (!parsed) continue + const ms = parsed.getTime() + if (ms < EARLIEST_PLAUSIBLE_MS || ms > now.getTime() + FUTURE_TOLERANCE_MS) continue + return parsed + } + return null +} diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts index 0e459335f1f..61ac44d2c7f 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts @@ -14,17 +14,17 @@ import { } from '@sim/testing' import { generateShortId } from '@sim/utils/id' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { isConnectorRunnableStatus } from '@/lib/knowledge/connectors/sync-engine' import { classifySuspectListing, evaluateListingSafety, - isConnectorRunnableStatus, isStuckDocumentSweepEligible, mergeHydratedDocument, mergeHydratedSkippedDocument, type PreviousListingObservation, selectStuckDocumentSweepCandidates, stuckDocumentSweepAgeAnchor, -} from '@/lib/knowledge/connectors/sync-engine' +} from '@/lib/knowledge/connectors/sync-primitives' import type { ExternalDocument, SyncResult } from '@/connectors/types' vi.mock('drizzle-orm', () => drizzleOrmMock) @@ -90,14 +90,14 @@ describe('isConnectorRunnableStatus', () => { describe('shouldReconcileDeletions', () => { it('runs on a clean full listing', async () => { - const { shouldReconcileDeletions } = await import('@/lib/knowledge/connectors/sync-engine') + const { shouldReconcileDeletions } = await import('@/lib/knowledge/connectors/sync-primitives') expect(shouldReconcileDeletions(false, {}, undefined)).toBe(true) expect(shouldReconcileDeletions(false, undefined, undefined)).toBe(true) }) it('never runs on incremental syncs', async () => { - const { shouldReconcileDeletions } = await import('@/lib/knowledge/connectors/sync-engine') + const { shouldReconcileDeletions } = await import('@/lib/knowledge/connectors/sync-primitives') expect(shouldReconcileDeletions(true, {}, undefined)).toBe(false) expect(shouldReconcileDeletions(true, {}, true)).toBe(false) @@ -105,20 +105,20 @@ describe('shouldReconcileDeletions', () => { }) it('skips when a connector capped the listing', async () => { - const { shouldReconcileDeletions } = await import('@/lib/knowledge/connectors/sync-engine') + const { shouldReconcileDeletions } = await import('@/lib/knowledge/connectors/sync-primitives') expect(shouldReconcileDeletions(false, { listingCapped: true }, undefined)).toBe(false) expect(shouldReconcileDeletions(false, { listingCapped: true }, false)).toBe(false) }) it('lets a forced fullSync override a connector cap', async () => { - const { shouldReconcileDeletions } = await import('@/lib/knowledge/connectors/sync-engine') + const { shouldReconcileDeletions } = await import('@/lib/knowledge/connectors/sync-primitives') expect(shouldReconcileDeletions(false, { listingCapped: true }, true)).toBe(true) }) it('never runs when the engine truncated pagination, even on a forced fullSync', async () => { - const { shouldReconcileDeletions } = await import('@/lib/knowledge/connectors/sync-engine') + const { shouldReconcileDeletions } = await import('@/lib/knowledge/connectors/sync-primitives') expect(shouldReconcileDeletions(false, { listingTruncated: true }, undefined)).toBe(false) expect(shouldReconcileDeletions(false, { listingTruncated: true }, true)).toBe(false) @@ -128,7 +128,7 @@ describe('shouldReconcileDeletions', () => { }) it('never runs when provider pagination is non-authoritative', async () => { - const { shouldReconcileDeletions } = await import('@/lib/knowledge/connectors/sync-engine') + const { shouldReconcileDeletions } = await import('@/lib/knowledge/connectors/sync-primitives') expect(shouldReconcileDeletions(false, { reconciliationUnsafe: true }, undefined)).toBe(false) expect(shouldReconcileDeletions(false, { reconciliationUnsafe: true }, true)).toBe(false) @@ -139,7 +139,7 @@ describe('shouldRunIncrementalSync', () => { const lastSyncAt = '2026-07-01T00:00:00.000Z' it('runs incrementally when everything is eligible', async () => { - const { shouldRunIncrementalSync } = await import('@/lib/knowledge/connectors/sync-engine') + const { shouldRunIncrementalSync } = await import('@/lib/knowledge/connectors/sync-primitives') expect( shouldRunIncrementalSync(true, 'incremental', undefined, undefined, false, lastSyncAt) @@ -147,7 +147,7 @@ describe('shouldRunIncrementalSync', () => { }) it('never runs incrementally when the connector does not support it', async () => { - const { shouldRunIncrementalSync } = await import('@/lib/knowledge/connectors/sync-engine') + const { shouldRunIncrementalSync } = await import('@/lib/knowledge/connectors/sync-primitives') expect( shouldRunIncrementalSync(false, 'incremental', undefined, undefined, false, lastSyncAt) @@ -155,7 +155,7 @@ describe('shouldRunIncrementalSync', () => { }) it('never runs incrementally when the connector is configured for full syncs', async () => { - const { shouldRunIncrementalSync } = await import('@/lib/knowledge/connectors/sync-engine') + const { shouldRunIncrementalSync } = await import('@/lib/knowledge/connectors/sync-primitives') expect(shouldRunIncrementalSync(true, 'full', undefined, undefined, false, lastSyncAt)).toBe( false @@ -163,7 +163,7 @@ describe('shouldRunIncrementalSync', () => { }) it('never runs incrementally on a forced fullSync or rehydrate', async () => { - const { shouldRunIncrementalSync } = await import('@/lib/knowledge/connectors/sync-engine') + const { shouldRunIncrementalSync } = await import('@/lib/knowledge/connectors/sync-primitives') expect(shouldRunIncrementalSync(true, 'incremental', true, undefined, false, lastSyncAt)).toBe( false @@ -174,7 +174,7 @@ describe('shouldRunIncrementalSync', () => { }) it('never runs incrementally before the first sync', async () => { - const { shouldRunIncrementalSync } = await import('@/lib/knowledge/connectors/sync-engine') + const { shouldRunIncrementalSync } = await import('@/lib/knowledge/connectors/sync-primitives') expect(shouldRunIncrementalSync(true, 'incremental', undefined, undefined, false, null)).toBe( false @@ -182,7 +182,7 @@ describe('shouldRunIncrementalSync', () => { }) it('forces a full listing whenever pending-removal documents exist, so they get a resurrect-or-confirm decision', async () => { - const { shouldRunIncrementalSync } = await import('@/lib/knowledge/connectors/sync-engine') + const { shouldRunIncrementalSync } = await import('@/lib/knowledge/connectors/sync-primitives') expect( shouldRunIncrementalSync(true, 'incremental', undefined, undefined, true, lastSyncAt) @@ -195,7 +195,9 @@ describe('partitionSyncReconciliation', () => { const noFailures = new Set() it('marks a live document missing from the listing as pending removal, not hard-deleted', async () => { - const { partitionSyncReconciliation } = await import('@/lib/knowledge/connectors/sync-engine') + const { partitionSyncReconciliation } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) const result = partitionSyncReconciliation([live('a')], [], new Set(), noFailures, undefined) @@ -203,7 +205,9 @@ describe('partitionSyncReconciliation', () => { }) it('hard-deletes a document already pending removal that is still absent', async () => { - const { partitionSyncReconciliation } = await import('@/lib/knowledge/connectors/sync-engine') + const { partitionSyncReconciliation } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) const result = partitionSyncReconciliation([], [live('a')], new Set(), noFailures, undefined) @@ -211,7 +215,9 @@ describe('partitionSyncReconciliation', () => { }) it('resurrects a pending-removal document that reappears in the listing', async () => { - const { partitionSyncReconciliation } = await import('@/lib/knowledge/connectors/sync-engine') + const { partitionSyncReconciliation } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) const result = partitionSyncReconciliation( [], @@ -225,7 +231,9 @@ describe('partitionSyncReconciliation', () => { }) it('leaves a document untouched when it is still present in the listing', async () => { - const { partitionSyncReconciliation } = await import('@/lib/knowledge/connectors/sync-engine') + const { partitionSyncReconciliation } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) const result = partitionSyncReconciliation( [live('a')], @@ -239,7 +247,9 @@ describe('partitionSyncReconciliation', () => { }) it('resurrects even on a forced fullSync', async () => { - const { partitionSyncReconciliation } = await import('@/lib/knowledge/connectors/sync-engine') + const { partitionSyncReconciliation } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) const result = partitionSyncReconciliation([], [live('a')], new Set(['a']), noFailures, true) @@ -247,7 +257,9 @@ describe('partitionSyncReconciliation', () => { }) it('hard-deletes both live and pending-removal documents immediately on a forced fullSync', async () => { - const { partitionSyncReconciliation } = await import('@/lib/knowledge/connectors/sync-engine') + const { partitionSyncReconciliation } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) const result = partitionSyncReconciliation( [live('a')], @@ -262,7 +274,9 @@ describe('partitionSyncReconciliation', () => { }) it('handles a mixed batch of every outcome in one pass', async () => { - const { partitionSyncReconciliation } = await import('@/lib/knowledge/connectors/sync-engine') + const { partitionSyncReconciliation } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) const result = partitionSyncReconciliation( [live('kept'), live('newly-missing')], @@ -280,7 +294,9 @@ describe('partitionSyncReconciliation', () => { }) it('ignores documents with a null externalId', async () => { - const { partitionSyncReconciliation } = await import('@/lib/knowledge/connectors/sync-engine') + const { partitionSyncReconciliation } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) const result = partitionSyncReconciliation( [live('a', null)], @@ -294,7 +310,9 @@ describe('partitionSyncReconciliation', () => { }) it('does not resurrect a reappearing document whose content refresh failed', async () => { - const { partitionSyncReconciliation } = await import('@/lib/knowledge/connectors/sync-engine') + const { partitionSyncReconciliation } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) const result = partitionSyncReconciliation( [], @@ -308,7 +326,9 @@ describe('partitionSyncReconciliation', () => { }) it('still refuses to resurrect a failed refresh even on a forced fullSync', async () => { - const { partitionSyncReconciliation } = await import('@/lib/knowledge/connectors/sync-engine') + const { partitionSyncReconciliation } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) const result = partitionSyncReconciliation( [], @@ -322,7 +342,9 @@ describe('partitionSyncReconciliation', () => { }) it('resurrects the ones that succeeded while excluding the one that failed', async () => { - const { partitionSyncReconciliation } = await import('@/lib/knowledge/connectors/sync-engine') + const { partitionSyncReconciliation } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) const result = partitionSyncReconciliation( [], @@ -339,7 +361,7 @@ describe('partitionSyncReconciliation', () => { describe('filterStillOwnedReconciliationIds', () => { it('keeps ids present in the ownership snapshot', async () => { const { filterStillOwnedReconciliationIds } = await import( - '@/lib/knowledge/connectors/sync-engine' + '@/lib/knowledge/connectors/sync-primitives' ) const result = filterStillOwnedReconciliationIds(['a'], ['b'], ['c'], new Set(['a', 'b', 'c'])) @@ -349,7 +371,7 @@ describe('filterStillOwnedReconciliationIds', () => { it('drops ids a concurrent connector-delete already detached', async () => { const { filterStillOwnedReconciliationIds } = await import( - '@/lib/knowledge/connectors/sync-engine' + '@/lib/knowledge/connectors/sync-primitives' ) const result = filterStillOwnedReconciliationIds(['a'], ['b'], ['c'], new Set(['a'])) @@ -359,7 +381,7 @@ describe('filterStillOwnedReconciliationIds', () => { it('returns all-empty lists when nothing is still owned', async () => { const { filterStillOwnedReconciliationIds } = await import( - '@/lib/knowledge/connectors/sync-engine' + '@/lib/knowledge/connectors/sync-primitives' ) const result = filterStillOwnedReconciliationIds(['a'], ['b'], ['c'], new Set()) @@ -380,7 +402,7 @@ describe('resolveTagMapping', () => { priority: 'High', }) - const { resolveTagMapping } = await import('@/lib/knowledge/connectors/sync-engine') + const { resolveTagMapping } = await import('@/lib/knowledge/connectors/sync-persistence') const result = resolveTagMapping( 'jira', @@ -402,7 +424,7 @@ describe('resolveTagMapping', () => { }) it('returns undefined when connector has no mapTags', async () => { - const { resolveTagMapping } = await import('@/lib/knowledge/connectors/sync-engine') + const { resolveTagMapping } = await import('@/lib/knowledge/connectors/sync-persistence') const result = resolveTagMapping( 'no-tags', @@ -416,7 +438,7 @@ describe('resolveTagMapping', () => { }) it('returns undefined when connector type is unknown', async () => { - const { resolveTagMapping } = await import('@/lib/knowledge/connectors/sync-engine') + const { resolveTagMapping } = await import('@/lib/knowledge/connectors/sync-persistence') const result = resolveTagMapping('unknown', { key: 'value' }, {}) @@ -426,7 +448,7 @@ describe('resolveTagMapping', () => { it('returns undefined when no tagSlotMapping in sourceConfig', async () => { mockMapTags.mockReturnValue({ issueType: 'Bug' }) - const { resolveTagMapping } = await import('@/lib/knowledge/connectors/sync-engine') + const { resolveTagMapping } = await import('@/lib/knowledge/connectors/sync-persistence') const result = resolveTagMapping('jira', { issueType: 'Bug' }, {}) @@ -439,7 +461,7 @@ describe('resolveTagMapping', () => { status: undefined, }) - const { resolveTagMapping } = await import('@/lib/knowledge/connectors/sync-engine') + const { resolveTagMapping } = await import('@/lib/knowledge/connectors/sync-persistence') const result = resolveTagMapping( 'jira', @@ -463,7 +485,7 @@ describe('resolveTagMapping', () => { it('returns undefined when sourceConfig is undefined', async () => { mockMapTags.mockReturnValue({ issueType: 'Bug' }) - const { resolveTagMapping } = await import('@/lib/knowledge/connectors/sync-engine') + const { resolveTagMapping } = await import('@/lib/knowledge/connectors/sync-persistence') const result = resolveTagMapping('jira', { issueType: 'Bug' }, undefined) @@ -475,14 +497,14 @@ describe('classifyExternalDoc', () => { const base = { content: 'hello', contentDeferred: false, contentHash: 'h1' } it('records a new skipped file as a failed row', async () => { - const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-engine') + const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-primitives') expect( classifyExternalDoc({ ...base, content: '', skippedReason: 'too big' }, undefined) ).toEqual({ type: 'skip' }) }) it('keeps an already-indexed file as-is when it becomes skipped (last-known-good)', async () => { - const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-engine') + const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-primitives') expect( classifyExternalDoc( { ...base, content: '', skippedReason: 'too big' }, @@ -496,7 +518,7 @@ describe('classifyExternalDoc', () => { }) it('refreshes an existing skipped placeholder without turning it into a source failure', async () => { - const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-engine') + const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-primitives') expect( classifyExternalDoc( @@ -507,7 +529,7 @@ describe('classifyExternalDoc', () => { }) it('rehydrates a content-less placeholder even when its listing hash is unchanged', async () => { - const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-engine') + const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-primitives') expect( classifyExternalDoc( @@ -519,7 +541,7 @@ describe('classifyExternalDoc', () => { it('uses the same skip replacement rule after deferred hydration', async () => { const { shouldReplaceExistingWithSkippedDocument } = await import( - '@/lib/knowledge/connectors/sync-engine' + '@/lib/knowledge/connectors/sync-primitives' ) expect(shouldReplaceExistingWithSkippedDocument({ storageKey: null }, {})).toBe(true) @@ -535,7 +557,7 @@ describe('classifyExternalDoc', () => { }) it('replaces stale indexed content for an authoritative skip', async () => { - const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-engine') + const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-primitives') expect( classifyExternalDoc( @@ -551,12 +573,12 @@ describe('classifyExternalDoc', () => { }) it('drops empty non-deferred content', async () => { - const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-engine') + const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-primitives') expect(classifyExternalDoc({ ...base, content: ' ' }, undefined)).toEqual({ type: 'drop' }) }) it('adds new content and deferred stubs', async () => { - const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-engine') + const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-primitives') expect(classifyExternalDoc(base, undefined)).toEqual({ type: 'add' }) expect(classifyExternalDoc({ ...base, content: '', contentDeferred: true }, undefined)).toEqual( { type: 'add' } @@ -564,7 +586,7 @@ describe('classifyExternalDoc', () => { }) it('updates when the content hash changed and is unchanged otherwise', async () => { - const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-engine') + const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-primitives') expect(classifyExternalDoc(base, { id: 'doc-1', contentHash: 'old' })).toEqual({ type: 'update', existingId: 'doc-1', @@ -575,7 +597,7 @@ describe('classifyExternalDoc', () => { }) it('forces re-hydration of an unchanged deferred doc when forceRehydrate is set', async () => { - const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-engine') + const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-primitives') const deferred = { ...base, content: '', contentDeferred: true } // Same hash → normally unchanged, but forceRehydrate promotes it to update. expect(classifyExternalDoc(deferred, { id: 'doc-1', contentHash: 'h1' }, true)).toEqual({ @@ -585,7 +607,7 @@ describe('classifyExternalDoc', () => { }) it('does not force re-hydration of a non-deferred doc (content already final)', async () => { - const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-engine') + const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-primitives') // Ready (non-deferred) content with an unchanged hash stays unchanged even under forceRehydrate. expect(classifyExternalDoc(base, { id: 'doc-1', contentHash: 'h1' }, true)).toEqual({ type: 'unchanged', @@ -603,6 +625,7 @@ describe('connector content replacement processing state', () => { sourceConfig: {}, syncMode: 'full', syncIntervalMinutes: 1440, + accessMode: 'workspace', status: 'active', lastSyncAt: null, lastSyncDocCount: 1, @@ -690,31 +713,82 @@ describe('connector content replacement processing state', () => { }) }) +/** The run's lease as the persistence writes see it; the condition itself is opaque to the chain mock. */ +const lease = { stillHeld: () => ({ type: 'lease' }) as never } + describe('persistSkippedDocuments', () => { beforeEach(() => { vi.clearAllMocks() resetDbChainMock() }) + /** + * The heartbeat before a batch only proves the lease was held then; the + * write itself re-proves it inside its transaction, so a run reclaimed in + * between lands nothing over its replacement's. + */ + it('refuses to write once the run no longer holds its lease', async () => { + const { persistSkippedDocuments } = await import('@/lib/knowledge/connectors/sync-persistence') + const { SyncLockLostException } = await import('@/lib/knowledge/connectors/sync-lock') + queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb-1' }]) + queueTableRows(schemaMock.knowledgeConnector, []) + + await expect( + persistSkippedDocuments( + 'kb-1', + 'connector-1', + 'no-tags', + [ + { + type: 'skip', + extDoc: { + externalId: 'external-1', + title: 'Empty document', + content: '', + mimeType: 'text/plain', + contentHash: 'empty-hash', + skippedReason: 'Document contains no extractable text', + }, + }, + ], + undefined, + 'workspace', + lease + ) + ).rejects.toBeInstanceOf(SyncLockLostException) + + expect(dbChainMockFns.for).toHaveBeenCalledWith('share') + expect(dbChainMockFns.values).not.toHaveBeenCalled() + }) + it('persists a new skipped document without dispatching processing', async () => { - const { persistSkippedDocuments } = await import('@/lib/knowledge/connectors/sync-engine') + const { persistSkippedDocuments } = await import('@/lib/knowledge/connectors/sync-persistence') queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb-1' }]) + queueTableRows(schemaMock.knowledgeConnector, [{ id: 'connector-1' }]) await expect( - persistSkippedDocuments('kb-1', 'connector-1', 'no-tags', [ - { - type: 'skip', - extDoc: { - externalId: 'external-1', - title: 'Empty document', - content: '', - mimeType: 'text/plain', - contentHash: 'empty-hash', - skippedReason: 'Document contains no extractable text', - skippedExistingDisposition: 'replace', + persistSkippedDocuments( + 'kb-1', + 'connector-1', + 'no-tags', + [ + { + type: 'skip', + extDoc: { + externalId: 'external-1', + title: 'Empty document', + content: '', + mimeType: 'text/plain', + contentHash: 'empty-hash', + skippedReason: 'Document contains no extractable text', + skippedExistingDisposition: 'replace', + }, }, - }, - ]) + ], + undefined, + 'workspace', + lease + ) ).resolves.toBe(1) expect(dbChainMockFns.values).toHaveBeenCalledWith([ @@ -730,28 +804,37 @@ describe('persistSkippedDocuments', () => { }) it('atomically replaces stale indexed content for an authoritative skip', async () => { - const { persistSkippedDocuments } = await import('@/lib/knowledge/connectors/sync-engine') + const { persistSkippedDocuments } = await import('@/lib/knowledge/connectors/sync-persistence') const oldFileUrl = '/api/files/serve/kb/old-document.txt?context=knowledge-base' queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb-1' }]) + queueTableRows(schemaMock.knowledgeConnector, [{ id: 'connector-1' }]) queueTableRows(schemaMock.document, [{ fileUrl: oldFileUrl }]) dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'doc-1' }]) await expect( - persistSkippedDocuments('kb-1', 'connector-1', 'no-tags', [ - { - type: 'skip', - existingId: 'doc-1', - extDoc: { - externalId: 'external-1', - title: 'Empty document', - content: '', - mimeType: 'text/plain', - contentHash: 'new-empty-hash', - skippedReason: 'Document contains no extractable text', - skippedExistingDisposition: 'replace', + persistSkippedDocuments( + 'kb-1', + 'connector-1', + 'no-tags', + [ + { + type: 'skip', + existingId: 'doc-1', + extDoc: { + externalId: 'external-1', + title: 'Empty document', + content: '', + mimeType: 'text/plain', + contentHash: 'new-empty-hash', + skippedReason: 'Document contains no extractable text', + skippedExistingDisposition: 'replace', + }, }, - }, - ]) + ], + undefined, + 'workspace', + lease + ) ).resolves.toBe(1) expect(dbChainMockFns.set).toHaveBeenCalledWith( @@ -778,26 +861,35 @@ describe('persistSkippedDocuments', () => { }) it('does not delete old storage when the authoritative replacement fails', async () => { - const { persistSkippedDocuments } = await import('@/lib/knowledge/connectors/sync-engine') + const { persistSkippedDocuments } = await import('@/lib/knowledge/connectors/sync-persistence') queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb-1' }]) + queueTableRows(schemaMock.knowledgeConnector, [{ id: 'connector-1' }]) queueTableRows(schemaMock.document, []) await expect( - persistSkippedDocuments('kb-1', 'connector-1', 'no-tags', [ - { - type: 'skip', - existingId: 'missing-doc', - extDoc: { - externalId: 'external-1', - title: 'Empty document', - content: '', - mimeType: 'text/plain', - contentHash: 'new-empty-hash', - skippedReason: 'Document contains no extractable text', - skippedExistingDisposition: 'replace', + persistSkippedDocuments( + 'kb-1', + 'connector-1', + 'no-tags', + [ + { + type: 'skip', + existingId: 'missing-doc', + extDoc: { + externalId: 'external-1', + title: 'Empty document', + content: '', + mimeType: 'text/plain', + contentHash: 'new-empty-hash', + skippedReason: 'Document contains no extractable text', + skippedExistingDisposition: 'replace', + }, }, - }, - ]) + ], + undefined, + 'workspace', + lease + ) ).rejects.toThrow('Document missing-doc is no longer active') expect(mockDeleteFile).not.toHaveBeenCalled() @@ -812,20 +904,27 @@ describe('persistSkippedRetryHashes', () => { }) it('updates only the retry hash for a last-known-good connector document', async () => { - const { classifyExternalDoc, persistSkippedRetryHashes } = await import( - '@/lib/knowledge/connectors/sync-engine' + const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-primitives') + const { persistSkippedRetryHashes } = await import( + '@/lib/knowledge/connectors/sync-persistence' ) queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb-1' }]) + queueTableRows(schemaMock.knowledgeConnector, [{ id: 'connector-1' }]) dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'doc-1' }]) await expect( - persistSkippedRetryHashes('kb-1', 'connector-1', [ - { - existingId: 'doc-1', - externalId: 'page-1', - contentHash: 'notion:retry:v1:page-1', - }, - ]) + persistSkippedRetryHashes( + 'kb-1', + 'connector-1', + [ + { + existingId: 'doc-1', + externalId: 'page-1', + contentHash: 'notion:retry:v1:page-1', + }, + ], + lease + ) ).resolves.toEqual([]) expect(dbChainMockFns.set).toHaveBeenCalledOnce() @@ -846,23 +945,31 @@ describe('persistSkippedRetryHashes', () => { }) it('commits live retry hashes when another document is no longer a connector target', async () => { - const { persistSkippedRetryHashes } = await import('@/lib/knowledge/connectors/sync-engine') + const { persistSkippedRetryHashes } = await import( + '@/lib/knowledge/connectors/sync-persistence' + ) queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb-1' }]) + queueTableRows(schemaMock.knowledgeConnector, [{ id: 'connector-1' }]) dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'live-doc' }]).mockResolvedValueOnce([]) await expect( - persistSkippedRetryHashes('kb-1', 'connector-1', [ - { - existingId: 'live-doc', - externalId: 'live-page', - contentHash: 'notion:retry:v1:live-page', - }, - { - existingId: 'detached-doc', - externalId: 'detached-page', - contentHash: 'notion:retry:v1:detached-page', - }, - ]) + persistSkippedRetryHashes( + 'kb-1', + 'connector-1', + [ + { + existingId: 'live-doc', + externalId: 'live-page', + contentHash: 'notion:retry:v1:live-page', + }, + { + existingId: 'detached-doc', + externalId: 'detached-page', + contentHash: 'notion:retry:v1:detached-page', + }, + ], + lease + ) ).resolves.toEqual(['detached-page']) expect(dbChainMockFns.set).toHaveBeenCalledWith({ @@ -899,7 +1006,7 @@ describe('chunkOpsByByteBudget', () => { }) it('batches small ops up to the count cap', async () => { - const { chunkOpsByByteBudget } = await import('@/lib/knowledge/connectors/sync-engine') + const { chunkOpsByByteBudget } = await import('@/lib/knowledge/connectors/sync-primitives') const chunks = chunkOpsByByteBudget( Array.from({ length: 7 }, () => addOp(1024)), 64 * MB, @@ -909,20 +1016,20 @@ describe('chunkOpsByByteBudget', () => { }) it('isolates a file larger than the budget into its own chunk', async () => { - const { chunkOpsByByteBudget } = await import('@/lib/knowledge/connectors/sync-engine') + const { chunkOpsByByteBudget } = await import('@/lib/knowledge/connectors/sync-primitives') const chunks = chunkOpsByByteBudget([addOp(100 * MB), addOp(1024)], 64 * MB, 5) expect(chunks.map((c) => c.length)).toEqual([1, 1]) }) it('caps summed bytes per chunk for medium files', async () => { - const { chunkOpsByByteBudget } = await import('@/lib/knowledge/connectors/sync-engine') + const { chunkOpsByByteBudget } = await import('@/lib/knowledge/connectors/sync-primitives') // 40 + 40 = 80 MB exceeds the 64 MB budget, so they split. const chunks = chunkOpsByByteBudget([addOp(40 * MB), addOp(40 * MB)], 64 * MB, 5) expect(chunks.map((c) => c.length)).toEqual([1, 1]) }) it('treats skip ops as zero bytes so they do not consume the budget', async () => { - const { chunkOpsByByteBudget } = await import('@/lib/knowledge/connectors/sync-engine') + const { chunkOpsByByteBudget } = await import('@/lib/knowledge/connectors/sync-primitives') const chunks = chunkOpsByByteBudget( [skipOp(100 * MB), skipOp(100 * MB), addOp(1024)], 64 * MB, @@ -938,7 +1045,7 @@ describe('connector sync working-set bounds', () => { CONNECTOR_SYNC_MAX_CORPUS_DOCUMENTS, sourcePageFitsSyncWorkingSet, syncWorkingSetQueryLimit, - } = await import('@/lib/knowledge/connectors/sync-engine') + } = await import('@/lib/knowledge/connectors/sync-primitives') expect(syncWorkingSetQueryLimit(0)).toBe(CONNECTOR_SYNC_MAX_CORPUS_DOCUMENTS + 1) expect(syncWorkingSetQueryLimit(CONNECTOR_SYNC_MAX_CORPUS_DOCUMENTS - 25)).toBe(26) @@ -949,7 +1056,7 @@ describe('connector sync working-set bounds', () => { it('counts retained source payload in UTF-8 bytes', async () => { const { addSourcePagePayloadBytes, CONNECTOR_SYNC_MAX_SOURCE_PAYLOAD_BYTES } = await import( - '@/lib/knowledge/connectors/sync-engine' + '@/lib/knowledge/connectors/sync-primitives' ) const document = { externalId: '', @@ -978,6 +1085,7 @@ describe('executeSync working-set overflow admission', () => { sourceConfig: {}, syncMode: 'full', syncIntervalMinutes: 1440, + accessMode: 'workspace', status: 'active', lastSyncAt: null, lastSyncDocCount: null, @@ -1074,9 +1182,10 @@ describe('executeSync working-set overflow admission', () => { } it('rejects overflow on a later source page before classification or document work', async () => { - const { CONNECTOR_SYNC_MAX_CORPUS_DOCUMENTS, executeSync } = await import( - '@/lib/knowledge/connectors/sync-engine' + const { CONNECTOR_SYNC_MAX_CORPUS_DOCUMENTS } = await import( + '@/lib/knowledge/connectors/sync-primitives' ) + const { executeSync } = await import('@/lib/knowledge/connectors/sync-engine') const retained = trackedSourceDocument('retained') const overflow = trackedSourceDocument('overflow') mockListDocuments @@ -1145,9 +1254,10 @@ describe('executeSync working-set overflow admission', () => { ])( 'rejects overflow in the sequential $population population before classification or document work', async ({ expectedDocumentReads, populations }) => { - const { CONNECTOR_SYNC_MAX_CORPUS_DOCUMENTS, executeSync } = await import( - '@/lib/knowledge/connectors/sync-engine' + const { CONNECTOR_SYNC_MAX_CORPUS_DOCUMENTS } = await import( + '@/lib/knowledge/connectors/sync-primitives' ) + const { executeSync } = await import('@/lib/knowledge/connectors/sync-engine') const listed = trackedSourceDocument('new-source-document') mockListDocuments.mockResolvedValue({ documents: [listed.document], hasMore: false }) for (const population of populations(CONNECTOR_SYNC_MAX_CORPUS_DOCUMENTS)) { @@ -1179,6 +1289,7 @@ describe('executeSync deferred hydration rate limits', () => { sourceConfig: {}, syncMode: 'full', syncIntervalMinutes: 1440, + accessMode: 'workspace', status: 'active', lastSyncAt: null, lastSyncDocCount: null, @@ -1481,7 +1592,9 @@ describe('mergeHydratedSkippedDocument', () => { describe('requireHydratedListedDocument', () => { it('turns ambiguous null hydration into a sync failure instead of a silent drop', async () => { - const { requireHydratedListedDocument } = await import('@/lib/knowledge/connectors/sync-engine') + const { requireHydratedListedDocument } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) expect(() => requireHydratedListedDocument(null, 'listed-1')).toThrow( 'Connector returned no content for listed document listed-1' @@ -1489,7 +1602,9 @@ describe('requireHydratedListedDocument', () => { }) it('passes through a hydrated document', async () => { - const { requireHydratedListedDocument } = await import('@/lib/knowledge/connectors/sync-engine') + const { requireHydratedListedDocument } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) const hydrated: ExternalDocument = { externalId: 'listed-1', title: 'Listed', @@ -1504,7 +1619,7 @@ describe('requireHydratedListedDocument', () => { describe('recordUnverifiedExistingRefresh', () => { it('keeps last-known-good content while holding the incremental watermark', async () => { const { recordUnverifiedExistingRefresh } = await import( - '@/lib/knowledge/connectors/sync-engine' + '@/lib/knowledge/connectors/sync-primitives' ) const result = { docsFailed: 0 } const failedExternalIds = new Set() @@ -1517,7 +1632,7 @@ describe('recordUnverifiedExistingRefresh', () => { it('counts one document once if multiple unusable signals converge', async () => { const { recordUnverifiedExistingRefresh } = await import( - '@/lib/knowledge/connectors/sync-engine' + '@/lib/knowledge/connectors/sync-primitives' ) const result = { docsFailed: 0 } const failedExternalIds = new Set() @@ -1858,7 +1973,7 @@ describe('selectStuckDocumentSweepCandidates', () => { describe('resolveReconciliationDeleteCap', () => { it('scales with the owned corpus above the absolute floor', async () => { const { resolveReconciliationDeleteCap } = await import( - '@/lib/knowledge/connectors/sync-engine' + '@/lib/knowledge/connectors/sync-primitives' ) expect(resolveReconciliationDeleteCap(1000)).toBe(250) @@ -1868,7 +1983,7 @@ describe('resolveReconciliationDeleteCap', () => { it('never drops below the absolute floor on a small corpus', async () => { const { resolveReconciliationDeleteCap } = await import( - '@/lib/knowledge/connectors/sync-engine' + '@/lib/knowledge/connectors/sync-primitives' ) expect(resolveReconciliationDeleteCap(0)).toBe(25) @@ -1879,7 +1994,7 @@ describe('resolveReconciliationDeleteCap', () => { it('honours an override that raises or lowers the cap', async () => { const { resolveReconciliationDeleteCap } = await import( - '@/lib/knowledge/connectors/sync-engine' + '@/lib/knowledge/connectors/sync-primitives' ) expect(resolveReconciliationDeleteCap(1000, { maxRatio: 0.9 })).toBe(900) @@ -1893,7 +2008,9 @@ describe('capReconciliationDeletions', () => { Array.from({ length: count }, (_, i) => `${prefix}-${i}`) it('passes a request exactly at the cap through untouched', async () => { - const { capReconciliationDeletions } = await import('@/lib/knowledge/connectors/sync-engine') + const { capReconciliationDeletions } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) const soft = ids('soft', 250) const result = capReconciliationDeletions(soft, [], 1000, false) @@ -1905,7 +2022,9 @@ describe('capReconciliationDeletions', () => { }) it('holds a request one document over the cap', async () => { - const { capReconciliationDeletions } = await import('@/lib/knowledge/connectors/sync-engine') + const { capReconciliationDeletions } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) const result = capReconciliationDeletions(ids('soft', 251), [], 1000, false) @@ -1915,7 +2034,9 @@ describe('capReconciliationDeletions', () => { }) it('returns empty arrays — not the inputs — when held', async () => { - const { capReconciliationDeletions } = await import('@/lib/knowledge/connectors/sync-engine') + const { capReconciliationDeletions } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) const result = capReconciliationDeletions(ids('soft', 300), ids('hard', 300), 1000, false) @@ -1925,7 +2046,9 @@ describe('capReconciliationDeletions', () => { }) it('caps each generation separately rather than summing them', async () => { - const { capReconciliationDeletions } = await import('@/lib/knowledge/connectors/sync-engine') + const { capReconciliationDeletions } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) /** * Hard deletes are the previous generation's soft deletes, already gated by @@ -1940,7 +2063,9 @@ describe('capReconciliationDeletions', () => { }) it('holds only the generation that breached the cap', async () => { - const { capReconciliationDeletions } = await import('@/lib/knowledge/connectors/sync-engine') + const { capReconciliationDeletions } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) const hard = ids('hard', 100) const result = capReconciliationDeletions(ids('soft', 400), hard, 1000, false) @@ -1953,7 +2078,9 @@ describe('capReconciliationDeletions', () => { }) it('is bypassed by a forced fullSync, in both generations', async () => { - const { capReconciliationDeletions } = await import('@/lib/knowledge/connectors/sync-engine') + const { capReconciliationDeletions } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) const hard = ids('hard', 1000) const hardOnly = capReconciliationDeletions([], hard, 1000, true) @@ -1974,14 +2101,18 @@ describe('capReconciliationDeletions', () => { }) it('applies the small-corpus floor rather than the ratio', async () => { - const { capReconciliationDeletions } = await import('@/lib/knowledge/connectors/sync-engine') + const { capReconciliationDeletions } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) expect(capReconciliationDeletions(ids('soft', 25), [], 8, false).held).toBe(false) expect(capReconciliationDeletions(ids('soft', 26), [], 8, false).held).toBe(true) }) it('honours an override that raises or lowers the cap', async () => { - const { capReconciliationDeletions } = await import('@/lib/knowledge/connectors/sync-engine') + const { capReconciliationDeletions } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) expect(capReconciliationDeletions(ids('s', 400), [], 1000, false, { maxRatio: 0.5 }).held).toBe( false @@ -1996,7 +2127,9 @@ describe('capReconciliationDeletions', () => { describe('steady churn', () => { it('reaches a stable state instead of ratcheting shut', async () => { - const { capReconciliationDeletions } = await import('@/lib/knowledge/connectors/sync-engine') + const { capReconciliationDeletions } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) /** * 1,000 documents at 15% churn against a cap of 250. Under one summed cap: @@ -2018,7 +2151,9 @@ describe('capReconciliationDeletions', () => { describe('confirmed data-loss shapes', () => { it('holds a partial outage that returns half a 1000-document corpus', async () => { - const { capReconciliationDeletions } = await import('@/lib/knowledge/connectors/sync-engine') + const { capReconciliationDeletions } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) const result = capReconciliationDeletions(ids('missing', 500), [], 1000, false) @@ -2028,7 +2163,9 @@ describe('capReconciliationDeletions', () => { }) it('holds an externalId derivation change that orphans the whole corpus', async () => { - const { capReconciliationDeletions } = await import('@/lib/knowledge/connectors/sync-engine') + const { capReconciliationDeletions } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) const result = capReconciliationDeletions(ids('old-key', 1000), [], 1000, false) @@ -2041,7 +2178,7 @@ describe('capReconciliationDeletions', () => { describe('resolvePreviousOwnedCount', () => { it('falls back to the current owned count when the recorded count collapsed', async () => { - const { resolvePreviousOwnedCount } = await import('@/lib/knowledge/connectors/sync-engine') + const { resolvePreviousOwnedCount } = await import('@/lib/knowledge/connectors/sync-primitives') // lastSyncDocCount excludes tombstones, so a soft-delete pass drives it to 0. expect(resolvePreviousOwnedCount(0, 500)).toBe(500) @@ -2050,7 +2187,7 @@ describe('resolvePreviousOwnedCount', () => { }) it('keeps the recorded count when it is the larger observation', async () => { - const { resolvePreviousOwnedCount } = await import('@/lib/knowledge/connectors/sync-engine') + const { resolvePreviousOwnedCount } = await import('@/lib/knowledge/connectors/sync-primitives') expect(resolvePreviousOwnedCount(800, 500)).toBe(800) expect(resolvePreviousOwnedCount(500, 500)).toBe(500) @@ -2063,7 +2200,9 @@ describe('partitionSyncReconciliation — user-excluded documents', () => { const noFailures = new Set() it('never hard-deletes an excluded document that is already pending removal', async () => { - const { partitionSyncReconciliation } = await import('@/lib/knowledge/connectors/sync-engine') + const { partitionSyncReconciliation } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) const result = partitionSyncReconciliation( [], @@ -2078,7 +2217,9 @@ describe('partitionSyncReconciliation — user-excluded documents', () => { }) it('still resurrects an excluded pending-removal document that reappears', async () => { - const { partitionSyncReconciliation } = await import('@/lib/knowledge/connectors/sync-engine') + const { partitionSyncReconciliation } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) /** * The assertion that rejects the select-level filter. Dropping excluded rows @@ -2099,7 +2240,9 @@ describe('partitionSyncReconciliation — user-excluded documents', () => { }) it('never soft-deletes an excluded live document absent from the listing', async () => { - const { partitionSyncReconciliation } = await import('@/lib/knowledge/connectors/sync-engine') + const { partitionSyncReconciliation } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) const result = partitionSyncReconciliation( [excluded('kept'), doc('gone')], @@ -2113,7 +2256,9 @@ describe('partitionSyncReconciliation — user-excluded documents', () => { }) it('exempts excluded documents from a forced fullSync purge too', async () => { - const { partitionSyncReconciliation } = await import('@/lib/knowledge/connectors/sync-engine') + const { partitionSyncReconciliation } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) const result = partitionSyncReconciliation( [excluded('kept-live'), doc('gone-live')], @@ -2129,7 +2274,9 @@ describe('partitionSyncReconciliation — user-excluded documents', () => { describe('connectorDocumentSyncTarget', () => { it('cannot refresh a detached, moved, excluded, or archived document', async () => { - const { connectorDocumentSyncTarget } = await import('@/lib/knowledge/connectors/sync-engine') + const { connectorDocumentSyncTarget } = await import( + '@/lib/knowledge/connectors/sync-persistence' + ) const condition = connectorDocumentSyncTarget('doc-1', 'kb-1', 'connector-1') for (const [column, value] of [ @@ -2158,14 +2305,14 @@ describe('connectorDocumentSyncTarget', () => { describe('countNonExcludedListed', () => { it('subtracts the excluded documents that appeared in the listing', async () => { - const { countNonExcludedListed } = await import('@/lib/knowledge/connectors/sync-engine') + const { countNonExcludedListed } = await import('@/lib/knowledge/connectors/sync-primitives') expect(countNonExcludedListed(new Set(['a', 'b', 'c']), new Set(['b']))).toBe(2) expect(countNonExcludedListed(new Set(['a', 'b']), new Set(['a', 'b']))).toBe(0) }) it('ignores excluded documents that were not listed', async () => { - const { countNonExcludedListed } = await import('@/lib/knowledge/connectors/sync-engine') + const { countNonExcludedListed } = await import('@/lib/knowledge/connectors/sync-primitives') expect(countNonExcludedListed(new Set(['a']), new Set(['x', 'y', 'z']))).toBe(1) expect(countNonExcludedListed(new Set(), new Set(['x']))).toBe(0) @@ -2173,7 +2320,7 @@ describe('countNonExcludedListed', () => { it('keeps the suspect-listing ratio on one population', async () => { const { classifySuspectListing, countNonExcludedListed } = await import( - '@/lib/knowledge/connectors/sync-engine' + '@/lib/knowledge/connectors/sync-primitives' ) /** @@ -2201,21 +2348,25 @@ describe('countDeletionEligibleOwned', () => { const excluded = (id: string) => ({ id, externalId: id, userExcluded: true }) it('does not let excluded tombstones inflate the denominator', async () => { - const { countDeletionEligibleOwned } = await import('@/lib/knowledge/connectors/sync-engine') + const { countDeletionEligibleOwned } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) expect(countDeletionEligibleOwned([doc('a')], [excluded('t1'), excluded('t2')])).toBe(1) expect(countDeletionEligibleOwned([doc('a')], [doc('t1'), excluded('t2')])).toBe(2) }) it('excludes user-excluded rows from the live side too', async () => { - const { countDeletionEligibleOwned } = await import('@/lib/knowledge/connectors/sync-engine') + const { countDeletionEligibleOwned } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) expect(countDeletionEligibleOwned([doc('a'), excluded('b')], [])).toBe(1) }) it('agrees with the numerator on which population it counts', async () => { const { classifySuspectListing, countDeletionEligibleOwned, countNonExcludedListed } = - await import('@/lib/knowledge/connectors/sync-engine') + await import('@/lib/knowledge/connectors/sync-primitives') /** * 100 live + 100 excluded tombstones. Counting the excluded tombstones would @@ -2238,7 +2389,9 @@ describe('countDeletionEligibleOwned', () => { describe('buildReconciliationHoldNotice', () => { it('places each count in its own role', async () => { - const { buildReconciliationHoldNotice } = await import('@/lib/knowledge/connectors/sync-engine') + const { buildReconciliationHoldNotice } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) /** * Asserted whole rather than by three independent `toContain` checks on @@ -2254,7 +2407,9 @@ describe('buildReconciliationHoldNotice', () => { }) it('does not claim withheld documents are indexed when only the purge was held', async () => { - const { buildReconciliationHoldNotice } = await import('@/lib/knowledge/connectors/sync-engine') + const { buildReconciliationHoldNotice } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) /** * A hard-only hold withholds documents a previous sync already tombstoned, @@ -2268,7 +2423,9 @@ describe('buildReconciliationHoldNotice', () => { }) it('names both consequences when both generations were held', async () => { - const { buildReconciliationHoldNotice } = await import('@/lib/knowledge/connectors/sync-engine') + const { buildReconciliationHoldNotice } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) const notice = buildReconciliationHoldNotice(900, 250, 1000, true, true) @@ -2277,7 +2434,9 @@ describe('buildReconciliationHoldNotice', () => { }) it('describes the cap as per generation, since a sync may spend it twice', async () => { - const { buildReconciliationHoldNotice } = await import('@/lib/knowledge/connectors/sync-engine') + const { buildReconciliationHoldNotice } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) // Saying "allowed in one sync" understated the real ceiling by 2x. expect(buildReconciliationHoldNotice(500, 250, 1000, true, false)).toContain( @@ -2286,7 +2445,9 @@ describe('buildReconciliationHoldNotice', () => { }) it('cannot be satisfied by swapping the withheld and cap counts', async () => { - const { buildReconciliationHoldNotice } = await import('@/lib/knowledge/connectors/sync-engine') + const { buildReconciliationHoldNotice } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) expect(buildReconciliationHoldNotice(500, 250, 1000, true, false)).not.toBe( buildReconciliationHoldNotice(250, 500, 1000, true, false) @@ -2459,7 +2620,7 @@ describe('sync lock lease', () => { }) it('opens the lease in the same statement that takes the lock', async () => { - const { buildSyncLockAcquisition } = await import('@/lib/knowledge/connectors/sync-engine') + const { buildSyncLockAcquisition } = await import('@/lib/knowledge/connectors/sync-lock') const values = buildSyncLockAcquisition('log-1', now) @@ -2607,6 +2768,8 @@ describe('completeSuccessfulSync', () => { queueTableRows(schemaMock.knowledgeConnector, [{ id: 'c-1' }]) queueTableRows(schemaMock.document, [{ count: 4 }]) dbChainMockFns.returning + /** The workspace ACL restore finds nothing drifted. */ + .mockResolvedValueOnce([]) .mockResolvedValueOnce([{ id: 'log-1' }]) .mockResolvedValueOnce([{ id: 'c-1' }]) @@ -2644,7 +2807,7 @@ describe('completeSuccessfulSync', () => { queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb-1' }]) queueTableRows(schemaMock.knowledgeConnector, [{ id: 'c-1' }]) queueTableRows(schemaMock.document, [{ count: 4 }]) - dbChainMockFns.returning.mockResolvedValueOnce([]) + dbChainMockFns.returning.mockResolvedValueOnce([]).mockResolvedValueOnce([]) await expect(completeSuccessfulSync('c-1', 'kb-1', 'log-1', 60, RESULT, null)).resolves.toBe( false @@ -2661,7 +2824,7 @@ describe('completeSuccessfulSync', () => { describe('stillHoldsSyncLock', () => { it('requires the connector to still be syncing', async () => { - const { stillHoldsSyncLock } = await import('@/lib/knowledge/connectors/sync-engine') + const { stillHoldsSyncLock } = await import('@/lib/knowledge/connectors/sync-lock') /** * Without this a run reclaimed by the stale sweep still writes its terminal @@ -2680,7 +2843,7 @@ describe('stillHoldsSyncLock', () => { }) it('still scopes to the connector and skips archived or deleted rows', async () => { - const { stillHoldsSyncLock } = await import('@/lib/knowledge/connectors/sync-engine') + const { stillHoldsSyncLock } = await import('@/lib/knowledge/connectors/sync-lock') const condition = stillHoldsSyncLock('c-1', 'run-a') @@ -2839,7 +3002,7 @@ describe('sync lock ownership across a reclaim and reacquire', () => { } it('rejects the reclaimed run A and admits the live run B', async () => { - const { stillHoldsSyncLock } = await import('@/lib/knowledge/connectors/sync-engine') + const { stillHoldsSyncLock } = await import('@/lib/knowledge/connectors/sync-lock') /** * A outlived the TTL, the reaper reclaimed its lock, and replacement B took @@ -2852,7 +3015,7 @@ describe('sync lock ownership across a reclaim and reacquire', () => { }) it('rejects a run whose lock was reclaimed with no replacement yet', async () => { - const { stillHoldsSyncLock } = await import('@/lib/knowledge/connectors/sync-engine') + const { stillHoldsSyncLock } = await import('@/lib/knowledge/connectors/sync-lock') const reclaimed = { ...rowHeldByB, @@ -2864,7 +3027,7 @@ describe('sync lock ownership across a reclaim and reacquire', () => { }) it('admits the run that still holds its own lock', async () => { - const { stillHoldsSyncLock } = await import('@/lib/knowledge/connectors/sync-engine') + const { stillHoldsSyncLock } = await import('@/lib/knowledge/connectors/sync-lock') const heldByA = { ...rowHeldByB, [schemaMock.knowledgeConnector.syncLockToken]: RUN_A } @@ -2872,7 +3035,7 @@ describe('sync lock ownership across a reclaim and reacquire', () => { }) it('rejects a run whose connector was paused mid-sync', async () => { - const { stillHoldsSyncLock } = await import('@/lib/knowledge/connectors/sync-engine') + const { stillHoldsSyncLock } = await import('@/lib/knowledge/connectors/sync-lock') const paused = { ...rowHeldByB, @@ -2893,7 +3056,7 @@ describe('sync lock ownership across a reclaim and reacquire', () => { describe('buildSyncLockAcquisition', () => { it('claims the lock and stamps ownership in one payload', async () => { - const { buildSyncLockAcquisition } = await import('@/lib/knowledge/connectors/sync-engine') + const { buildSyncLockAcquisition } = await import('@/lib/knowledge/connectors/sync-lock') const now = new Date('2026-08-20T00:00:00.000Z') const acquisition = buildSyncLockAcquisition('run-a', now) @@ -2910,7 +3073,7 @@ describe('buildSyncLockAcquisition', () => { describe('LOCKABLE_CONNECTOR_STATUSES', () => { it('refuses to start a run on a connector someone paused or disabled', async () => { - const { LOCKABLE_CONNECTOR_STATUSES } = await import('@/lib/knowledge/connectors/sync-engine') + const { LOCKABLE_CONNECTOR_STATUSES } = await import('@/lib/knowledge/connectors/sync-lock') /** * The queue outlives the decision to sync. A connector paused *after* its @@ -2935,21 +3098,21 @@ describe('LOCKABLE_CONNECTOR_STATUSES', () => { describe('shouldHeartbeatSyncLock', () => { it('beats once the interval has elapsed', async () => { - const { shouldHeartbeatSyncLock } = await import('@/lib/knowledge/connectors/sync-engine') + const { shouldHeartbeatSyncLock } = await import('@/lib/knowledge/connectors/sync-lock') expect(shouldHeartbeatSyncLock(1_000, 0, 1_000)).toBe(true) expect(shouldHeartbeatSyncLock(1_001, 0, 1_000)).toBe(true) }) it('does not beat before the interval has elapsed', async () => { - const { shouldHeartbeatSyncLock } = await import('@/lib/knowledge/connectors/sync-engine') + const { shouldHeartbeatSyncLock } = await import('@/lib/knowledge/connectors/sync-lock') expect(shouldHeartbeatSyncLock(999, 0, 1_000)).toBe(false) expect(shouldHeartbeatSyncLock(0, 0, 1_000)).toBe(false) }) it('defaults to an interval far below the reclaim TTL', async () => { - const { shouldHeartbeatSyncLock } = await import('@/lib/knowledge/connectors/sync-engine') + const { shouldHeartbeatSyncLock } = await import('@/lib/knowledge/connectors/sync-lock') const { CONNECTOR_SYNC_STALE_LOCK_TTL_MS, SYNC_LOCK_HEARTBEAT_INTERVAL_MS } = await import( '@/lib/knowledge/connectors/sync-limits' ) @@ -2972,7 +3135,7 @@ describe('heartbeatSyncLock', () => { }) it('extends the lock lease alone, under the run own lock guard', async () => { - const { heartbeatSyncLock } = await import('@/lib/knowledge/connectors/sync-engine') + const { heartbeatSyncLock } = await import('@/lib/knowledge/connectors/sync-lock') await heartbeatSyncLock('c-1', 'run-a') @@ -3000,7 +3163,7 @@ describe('heartbeatSyncLock', () => { }) it('reports a lost lock so the run can stop instead of racing its replacement', async () => { - const { heartbeatSyncLock } = await import('@/lib/knowledge/connectors/sync-engine') + const { heartbeatSyncLock } = await import('@/lib/knowledge/connectors/sync-lock') dbChainMockFns.returning.mockResolvedValueOnce([]) expect(await heartbeatSyncLock('c-1', 'run-a')).toBe(false) @@ -3010,7 +3173,7 @@ describe('heartbeatSyncLock', () => { }) it('can require the connector to remain live before destructive follow-up work', async () => { - const { heartbeatLiveSyncLock } = await import('@/lib/knowledge/connectors/sync-engine') + const { heartbeatLiveSyncLock } = await import('@/lib/knowledge/connectors/sync-lock') dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'c-1' }]) expect(await heartbeatLiveSyncLock('c-1', 'run-a')).toBe(true) @@ -3043,6 +3206,7 @@ describe('executeSync heartbeats during the listing phase', () => { sourceConfig: {}, syncMode: 'full', syncIntervalMinutes: 1440, + accessMode: 'workspace', status: 'active', lastSyncAt: null, lastSyncDocCount: null, @@ -3149,7 +3313,9 @@ describe('resolveStaleProcessingMinutes', () => { describe('SWEEPABLE_PROCESSING_STATUSES', () => { it('never includes a completed document', async () => { - const { SWEEPABLE_PROCESSING_STATUSES } = await import('@/lib/knowledge/connectors/sync-engine') + const { SWEEPABLE_PROCESSING_STATUSES } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) /** * The sweep reclaims by deleting embeddings and re-dispatching, so a @@ -3161,7 +3327,9 @@ describe('SWEEPABLE_PROCESSING_STATUSES', () => { }) it('covers every non-terminal state so nothing is stranded', async () => { - const { SWEEPABLE_PROCESSING_STATUSES } = await import('@/lib/knowledge/connectors/sync-engine') + const { SWEEPABLE_PROCESSING_STATUSES } = await import( + '@/lib/knowledge/connectors/sync-primitives' + ) const { DOCUMENT_PROCESSING_STATUSES } = await import('@/lib/knowledge/documents/types') const unreclaimable = DOCUMENT_PROCESSING_STATUSES.filter( @@ -3199,6 +3367,7 @@ describe('executeSync hard-delete reconciliation', () => { sourceConfig: {}, syncMode: 'full', syncIntervalMinutes: 1440, + accessMode: 'workspace', status: 'active', lastSyncAt: null, lastSyncDocCount: OWNED_DOC_COUNT, @@ -3313,8 +3482,9 @@ describe('executeSync hard-delete reconciliation', () => { }) it('bounds and orders the stuck-document sweep instead of draining a backlog at once', async () => { - const { executeSync, STUCK_RETRY_MAX_CANDIDATES_PER_SYNC } = await import( - '@/lib/knowledge/connectors/sync-engine' + const { executeSync } = await import('@/lib/knowledge/connectors/sync-engine') + const { STUCK_RETRY_MAX_CANDIDATES_PER_SYNC } = await import( + '@/lib/knowledge/connectors/sync-primitives' ) const { hardDeleteDocuments } = await import('@/lib/knowledge/documents/service') @@ -3516,6 +3686,7 @@ describe('executeSync terminal exits under a lost lock', () => { sourceConfig: {}, syncMode: 'full', syncIntervalMinutes: 1440, + accessMode: 'workspace', status: 'active', lastSyncAt: null, lastSyncDocCount: 0, diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.ts b/apps/sim/lib/knowledge/connectors/sync-engine.ts index c501a529593..06c76c14ad0 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.ts @@ -1,264 +1,69 @@ import { db } from '@sim/db' import { document, - embedding, knowledgeBase, knowledgeConnector, knowledgeConnectorSyncLog, } from '@sim/db/schema' import { createLogger } from '@sim/logger' -import { getErrorMessage, toError } from '@sim/utils/errors' +import { toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { randomInt } from '@sim/utils/random' -import { - and, - asc, - desc, - eq, - exists, - gt, - inArray, - isNotNull, - isNull, - lt, - ne, - or, - sql, -} from 'drizzle-orm' +import { and, asc, eq, exists, gt, inArray, isNotNull, isNull, sql } from 'drizzle-orm' import { decryptApiKey } from '@/lib/api-key/crypto' import { assertBillingAttributionSnapshot, type BillingAttributionSnapshot, } from '@/lib/billing/core/billing-attribution' -import { env, envNumber } from '@/lib/core/config/env' -import { getInternalApiBaseUrl } from '@/lib/core/utils/urls' import { resolveCredentialTokenIdentity } from '@/lib/credentials/access' import { CONNECTOR_AUTO_DISABLED_ERROR, CONNECTOR_FAILURE_BACKOFF_CAP_MINUTES, connectorFailureBackoffMinutes, MAX_CONSECUTIVE_FAILURES, - SYNC_LOCK_HEARTBEAT_INTERVAL_MS, } from '@/lib/knowledge/connectors/sync-limits' -import { DOCUMENT_PROCESSING_STALE_THRESHOLD_MS } from '@/lib/knowledge/documents/processing-timeouts.server' -import type { DocumentData } from '@/lib/knowledge/documents/service' import { - ConnectorSyncDeletionGuardError, - hardDeleteDocuments, - processDocumentsWithQueue, -} from '@/lib/knowledge/documents/service' + buildSyncLockAcquisition, + createContentSyncLease, + holdsSyncLockToken, + LOCKABLE_CONNECTOR_STATUSES, + SyncLockLostException, + stillHoldsSyncLock, +} from '@/lib/knowledge/connectors/sync-lock' import { - type DocumentProcessingStatus, - isDocumentProcessingStatus, - MAX_PROCESSING_ATTEMPTS, - QUEUED_DISPATCH_GRACE_MS, -} from '@/lib/knowledge/documents/types' + type KnowledgeBaseOwner, + restoreWorkspaceDocumentAcls, +} from '@/lib/knowledge/connectors/sync-persistence' +import { + ConnectorDeletedException, + ConnectorSyncCapacityError, + checkSyncTargetPresence, + classifyListing, + createSyncRunState, + loadOwnedCorpus, + processDocOps, + RETRY_WINDOW_DAYS, + reconcileDeletions, + runListingPass, + shouldRunIncrementalSync, + sweepStuckDocuments, +} from '@/lib/knowledge/connectors/sync-primitives' +import { hardDeleteDocuments } from '@/lib/knowledge/documents/service' import { getRetryAfterMs, isRateLimitError } from '@/lib/knowledge/documents/utils' import { refreshAccessTokenIfNeeded } from '@/lib/oauth/credential-service' -import { StorageService } from '@/lib/uploads' -import { buildStorageKeySegment } from '@/lib/uploads/core/storage-key' -import { deleteFile } from '@/lib/uploads/core/storage-service' -import { deleteFileMetadata } from '@/lib/uploads/server/metadata' -import { extractStorageKey } from '@/lib/uploads/utils/file-utils' import { CONNECTOR_REGISTRY } from '@/connectors/registry.server' -import type { - ConnectorAuthConfig, - DocumentTags, - ExternalDocument, - SyncResult, -} from '@/connectors/types' -import { hasIndexablePayload } from '@/connectors/utils' +import type { ConnectorAuthConfig, SyncResult } from '@/connectors/types' const logger = createLogger('ConnectorSyncEngine') const RATE_LIMIT_RETRY_JITTER_MAX_MS = 60_000 - -/** - * Raised when a run discovers mid-flight that it no longer holds its sync lock. - * - * Stops it doing hours of further work whose terminal write would be rejected, - * and — more importantly — stops it writing documents concurrently with the - * replacement run that took the lock. - */ -class SyncLockLostException extends Error { - constructor(connectorId: string) { - super(`Sync lock for connector ${connectorId} was reclaimed during sync`) - this.name = 'SyncLockLostException' - } -} - -class ConnectorDeletedException extends Error { - constructor(connectorId: string) { - super(`Connector ${connectorId} was deleted during sync`) - this.name = 'ConnectorDeletedException' - } -} - -const SYNC_BATCH_SIZE = 5 -/** Unknown deferred downloads run alone; actual connector files can reach this budget. */ -const DEFAULT_OP_SIZE_BYTES = 64 * 1024 * 1024 -/** - * Max summed source bytes hydrated/uploaded concurrently within a batch. Each - * in-flight file materializes as a content string plus an upload buffer, so this - * bounds peak worker memory: a few large files near the per-file cap are processed - * in smaller sub-chunks instead of all at once, while small files still process up - * to SYNC_BATCH_SIZE at a time. - */ -const CONTENT_INFLIGHT_BUDGET_BYTES = 64 * 1024 * 1024 -const MAX_PAGES = 500 -/** - * Maximum documents retained in either the source corpus or owned corpus. - * - * The engine needs the complete source identity set and the connector's complete - * owned-document set at the same time to distinguish adds from updates and to - * reconcile deletions safely. Page-count limits alone do not bound that working - * set: a connector page can contain many documents, and an incremental connector - * can accumulate a corpus much larger than its current page. The two corpora - * coexist, so the row-count peak is twice this value plus bounded - * maps and operation references. Crossing either per-corpus ceiling fails before - * document writes. - */ -export const CONNECTOR_SYNC_MAX_CORPUS_DOCUMENTS = 50_000 - -export const CONNECTOR_SYNC_MAX_SOURCE_PAYLOAD_BYTES = 256 * 1024 * 1024 -const MAX_SAFE_TITLE_LENGTH = 200 -/** - * How many stuck documents are re-dispatched per call. - * - * The retry backlog is unbounded, and on the in-process fallback path - * `processDocumentsWithQueue` parses, embeds, and indexes every document it is - * given before returning. Handing it the whole backlog made the retry a single - * await no heartbeat could interrupt; chunking gives the beat somewhere to run. - */ -const STUCK_RETRY_DISPATCH_CHUNK_SIZE = 25 - -/** - * How many stuck-document candidates one sync will consider. - * - * {@link STUCK_RETRY_DISPATCH_CHUNK_SIZE} paces the dispatch loop but does not - * bound it — the candidate query had no limit, so a connector carrying a large - * backlog dispatched the whole thing at once. One did: 2,959 documents enqueued - * in fifteen seconds onto a queue every workspace shares, at - * {@link PROCESSING_QUEUE_CONCURRENCY} concurrent runs. Nothing was - * double-billed — those documents were genuinely unindexed — but one connector - * monopolized the queue, and each dispatch mints a fresh `requestId`, so the - * Trigger.dev idempotency key differs every pass and none of it deduplicates. - * - * 200 keeps a single sync's contribution to roughly ten minutes of queue - * occupancy at the default concurrency. A backlog larger than this is not - * dropped: candidates are taken oldest-first and whatever is left stays - * eligible, so consecutive syncs drain it steadily instead of in one burst. - */ -export const STUCK_RETRY_MAX_CANDIDATES_PER_SYNC = 200 - -/** - * How many documents reconciliation hard-deletes per call. - * - * `hardDeleteDocuments` deletes storage objects, embeddings, and rows for its - * whole argument in serialized transactions, and a forced `fullSync` overriding - * a connector's listing cap can hand it tens of thousands of ids — one await - * spanning the widest gap between heartbeats in the sync, with the deletes - * themselves the slowest work in it. Chunking gives the beat somewhere to run, - * so a long purge stops looking dead to the reaper. Sized like the dispatch - * chunk above: small enough that a chunk cannot outlast the heartbeat interval, - * large enough that the per-call overhead stays negligible. - */ -const HARD_DELETE_CHUNK_SIZE = 25 const CONNECTOR_DELETION_CLEANUP_BATCH_SIZE = 250 -/** - * Concurrent `knowledge-process-document` runs, shared by every workspace. - * - * Read from the same env var the task itself is configured with rather than - * restated, so the drain estimate below cannot describe a queue depth the - * deployment does not actually run. - */ -const PROCESSING_QUEUE_CONCURRENCY = envNumber(env.KB_CONFIG_CONCURRENCY_LIMIT, 20) - -class ConnectorSyncCapacityError extends Error {} - -class ConnectorSyncWorkingSetLimitError extends ConnectorSyncCapacityError { - constructor(connectorId: string, scope: 'source listing' | 'owned corpus') { - super( - `Connector ${connectorId} ${scope} exceeds the safe per-corpus limit of ${CONNECTOR_SYNC_MAX_CORPUS_DOCUMENTS.toLocaleString()} documents. Narrow the configured source scope or set a connector document limit before syncing again.` - ) - this.name = 'ConnectorSyncWorkingSetLimitError' - } -} - -/** - * Returns a query's sentinel-inclusive limit for the remaining working-set - * budget. The extra row proves the corpus exceeded the cap without loading the - * rest of it. - */ -export function syncWorkingSetQueryLimit(rowsAlreadyLoaded: number): number { - return Math.max(CONNECTOR_SYNC_MAX_CORPUS_DOCUMENTS - rowsAlreadyLoaded, 0) + 1 -} - -export function sourcePageFitsSyncWorkingSet(rowsAlreadyLoaded: number, pageRows: number): boolean { - return rowsAlreadyLoaded + pageRows <= CONNECTOR_SYNC_MAX_CORPUS_DOCUMENTS -} - -function assertSyncWorkingSetWithinLimit( - connectorId: string, - rowsAlreadyLoaded: number, - rowsJustLoaded: number -): void { - if (rowsAlreadyLoaded + rowsJustLoaded > CONNECTOR_SYNC_MAX_CORPUS_DOCUMENTS) { - throw new ConnectorSyncWorkingSetLimitError(connectorId, 'owned corpus') - } -} - -function retainedExternalDocumentBytes(doc: ExternalDocument): number { - let bytes = Buffer.byteLength(doc.externalId) + Buffer.byteLength(doc.title) - bytes += Buffer.byteLength(doc.content ?? '') - bytes += Buffer.byteLength(doc.sourceUrl ?? '') - bytes += Buffer.byteLength(doc.contentHash ?? '') - if (doc.sourceFile?.bytes) bytes += doc.sourceFile.bytes.byteLength - try { - bytes += Buffer.byteLength(JSON.stringify(doc.metadata ?? {})) - } catch { - bytes += DEFAULT_OP_SIZE_BYTES - } - return bytes -} - -/** Fails listing before the engine retains an unbounded inline-content corpus. */ -export function addSourcePagePayloadBytes( - retainedBytes: number, - documents: ExternalDocument[] -): number { - let nextBytes = retainedBytes - for (const doc of documents) { - nextBytes += retainedExternalDocumentBytes(doc) - if (nextBytes > CONNECTOR_SYNC_MAX_SOURCE_PAYLOAD_BYTES) { - throw new ConnectorSyncCapacityError( - `Connector source listing exceeds the safe retained-payload limit of ${CONNECTOR_SYNC_MAX_SOURCE_PAYLOAD_BYTES.toLocaleString()} bytes. Use a narrower source scope or a deferred-content connector.` - ) - } - } - return nextBytes -} export { resolveStaleProcessingMinutes, worstCaseProcessingMinutes, } from '@/lib/knowledge/documents/types' -/** - * How long a document may sit in `processing` before the sweep treats its run as - * abandoned — and deletes its embeddings and re-dispatches it. - * - * DERIVED, not fixed at 45. The sweep reclaims by deleting live work, so this - * must exceed the longest a legitimate run can take. That bound is - * `KB_CONFIG_MAX_DURATION` x `KB_CONFIG_MAX_ATTEMPTS`, which an operator can - * raise: at the previous hard-coded 45, setting `KB_CONFIG_MAX_DURATION` above - * 900s silently made every long run look abandoned, so the sweep would delete - * the embeddings of documents that were still being indexed and bill a second - * pass. Deriving it keeps the invariant true at any configuration; the floor - * preserves today's value at the defaults. - */ -const STALE_PROCESSING_MINUTES = DOCUMENT_PROCESSING_STALE_THRESHOLD_MS / (60 * 1000) -const RETRY_WINDOW_DAYS = 7 const RUNNABLE_CONNECTOR_STATUSES = ['active', 'error'] as const /** Whether an automatic connector sync may begin from this persisted state. */ @@ -266,438 +71,6 @@ export function isConnectorRunnableStatus(status: string): boolean { return RUNNABLE_CONNECTOR_STATUSES.some((runnableStatus) => runnableStatus === status) } -/** - * Processing states the stuck-document sweep may reclaim from. - * - * One constant used by BOTH the candidate SELECT and the reset UPDATE. The - * UPDATE has to re-assert what the SELECT filtered on — the ownership re-check - * between them covers `connectorId` only, so a document that completed in that - * window would otherwise be reset and have its embeddings deleted. Sharing the - * list means the two cannot drift into disagreeing about what is reclaimable. - */ -export const SWEEPABLE_PROCESSING_STATUSES = ['pending', 'failed', 'processing'] as const - -/** The processing state the stuck-document sweep decides on, one row at a time. */ -export interface StuckDocumentSweepCandidate { - processingStatus: DocumentProcessingStatus - processingQueuedAt: Date | null - processingStartedAt: Date | null - processingDeferredUntil: Date | null - processingCompletedAt: Date | null - uploadedAt: Date -} - -/** - * Decides whether the sweep may reclaim one document — delete its embeddings, - * reset it, and dispatch it again. - * - * Since document processing is dispatched to `knowledge-process-document` - * rather than awaited inline, a document sits at `pending` from dispatch until - * a worker claims it; `processing` is only written once a worker has actually - * started. Reclaiming a `pending` document therefore risks racing a run that is - * still queued, which both duplicates its work and bills a second indexing - * pass, so queued documents get {@link QUEUED_DISPATCH_GRACE_MS} before they - * are considered lost — the same grace the user-facing retry waits out before - * it will admit a `pending` document. - * - * Queue wait is measured from `processingQueuedAt`, stamped in one place — - * `markDocumentsQueued`, which every dispatch funnels through, so the column - * always describes the attempt that is live right now. - * It falls back to `uploadedAt` when NULL, which covers a document dispatched - * by the sync that created it (`uploadedAt` then sits within that sync's own - * runtime, an over-estimate bounded by the one-hour sync ceiling) and rows - * written before the column existed. - * - * `failed` is not a terminal state and gets the same grace. `processDocumentAsync` - * records the failure and then rethrows, so `knowledge-process-document` retries - * it up to `maxAttempts` (3): between attempts the row reads `failed` while a - * live run is scheduled to pick it up again. The gap between attempts is bounded - * by the queue, not by run duration — a retried run re-enters the same queue - * behind the same global concurrency limit — so `maxDuration` x `maxAttempts` - * (30 minutes) and `STALE_PROCESSING_MINUTES` are both far too short to be safe - * here: on the very backlog this grace exists for, the next attempt starts hours - * after the last one ended. `failed` is therefore aged from - * `processingCompletedAt`, the instant the last attempt ended, which every - * failure write stamps. - * - * A document whose retries genuinely exhaust is still recovered: its final - * failure stops moving `processingCompletedAt`, so one grace later it becomes - * eligible and the next sync re-dispatches it. Recovery is delayed by the grace, - * never lost. The user-facing retry stays immediate — it writes `pending` and - * dispatches without consulting the sweep at all. - * - * The grace decides when a queued run may be superseded, but correctness does - * not depend on that timing judgment. Every task carries the queue stamp its - * dispatch installed and must match it before claiming or billing the row. A - * sweep clears the abandoned stamp before installing a new one, so a late old - * task declines while the replacement proceeds. Queue admission also claims - * only an empty stamp, preventing concurrent callers from charging or enqueuing - * two live generations for the same pending document. - */ -export function isStuckDocumentSweepEligible(doc: StuckDocumentSweepCandidate, now: Date): boolean { - switch (doc.processingStatus) { - case 'failed': { - const lastAttemptEndedAt = - doc.processingCompletedAt ?? doc.processingQueuedAt ?? doc.uploadedAt - return now.getTime() - lastAttemptEndedAt.getTime() > QUEUED_DISPATCH_GRACE_MS - } - case 'pending': { - if (doc.processingDeferredUntil) { - return now.getTime() - doc.processingDeferredUntil.getTime() > QUEUED_DISPATCH_GRACE_MS - } - const queuedAt = doc.processingQueuedAt ?? doc.uploadedAt - return now.getTime() - queuedAt.getTime() > QUEUED_DISPATCH_GRACE_MS - } - case 'processing': { - if (!doc.processingStartedAt) return true - return ( - now.getTime() - doc.processingStartedAt.getTime() > STALE_PROCESSING_MINUTES * 60 * 1000 - ) - } - // No `default`: a status added to DocumentProcessingStatus must fail - // type-check here rather than silently reading as "not eligible". - case 'completed': - return false - } -} - -export function stuckDocumentSweepAgeAnchor(doc: StuckDocumentSweepCandidate): Date { - switch (doc.processingStatus) { - case 'failed': - return doc.processingCompletedAt ?? doc.processingQueuedAt ?? doc.uploadedAt - case 'pending': - return doc.processingDeferredUntil ?? doc.processingQueuedAt ?? doc.uploadedAt - case 'processing': - return doc.processingStartedAt ?? new Date(0) - case 'completed': - return doc.uploadedAt - } -} - -export function selectStuckDocumentSweepCandidates< - T extends StuckDocumentSweepCandidate & { id: string }, ->(documents: T[], now: Date, limit = STUCK_RETRY_MAX_CANDIDATES_PER_SYNC): T[] { - return documents - .filter((doc) => isStuckDocumentSweepEligible(doc, now)) - .sort((left, right) => { - const ageOrder = - stuckDocumentSweepAgeAnchor(left).getTime() - stuckDocumentSweepAgeAnchor(right).getTime() - return ageOrder || left.id.localeCompare(right.id) - }) - .slice(0, limit) -} - -/** Sanitizes a document title for use in S3 storage keys. */ -function sanitizeStorageTitle(title: string): string { - return title.replace(/[^a-zA-Z0-9.-]/g, '_').slice(0, MAX_SAFE_TITLE_LENGTH) -} - -/** - * Sanitizes a source file's name for a storage key, keeping its extension. - * - * `sanitizeStorageTitle` truncates a long title outright, which for a source file - * would cut the extension off the end — and the extension is what - * `resolveStoredArtifactExtension` reads to pick a parser. Such a document would - * still parse correctly by falling back to its display name, but only by luck; - * preserving the suffix keeps the storage key authoritative for every file rather - * than for most of them. - */ -function sanitizeStorageFileName(fileName: string): string { - const dotIndex = fileName.lastIndexOf('.') - if (dotIndex <= 0) return sanitizeStorageTitle(fileName) - - const extension = sanitizeStorageTitle(fileName.slice(dotIndex)) - const base = sanitizeStorageTitle(fileName.slice(0, dotIndex)).slice( - 0, - Math.max(1, MAX_SAFE_TITLE_LENGTH - extension.length) - ) - return base + extension -} - -/** - * The bytes to store for a connector document, together with the name and type - * that describe them. - * - * The stored object must declare the format it actually holds, because - * `resolveStoredArtifactExtension` picks the parser off its storage key. A - * connector that hands over the source file keeps that file's own name and type, - * so the shared pipeline parses it exactly as an upload of the same file — which - * is what routes PDFs to OCR. A connector that extracted text itself stores - * `.txt`, since that is what the bytes now are; keeping the source extension - * there would re-parse extracted text as the original binary. - */ -function connectorStoredArtifact(extDoc: ExternalDocument): { - bytes: Buffer - fileName: string - mimeType: string -} { - if (extDoc.sourceFile) { - return { - bytes: extDoc.sourceFile.bytes, - fileName: sanitizeStorageFileName(extDoc.sourceFile.fileName), - mimeType: extDoc.sourceFile.mimeType, - } - } - return { - bytes: Buffer.from(extDoc.content, 'utf-8'), - fileName: `${sanitizeStorageTitle(extDoc.title)}.txt`, - mimeType: 'text/plain', - } -} -type KnowledgeBaseLockingTx = Pick - -type DocOp = - | { type: 'add'; extDoc: ExternalDocument } - | { type: 'update'; existingId: string; extDoc: ExternalDocument } - | { type: 'skip'; existingId?: string; extDoc: ExternalDocument } - -type DocClassification = - | { type: 'add' } - | { type: 'update'; existingId: string } - | { type: 'skip'; existingId?: string } - | { type: 'unchanged' } - | { type: 'drop' } - -export function shouldReplaceExistingWithSkippedDocument( - existing: { storageKey?: string | null }, - skipped: Pick -): boolean { - return existing.storageKey === null || skipped.skippedExistingDisposition === 'replace' -} - -/** - * Decides what a listed external document becomes during reconciliation. - * - * - `skip`: connector flagged it (e.g. too large) and it is not already indexed — - * record a visible `failed` document instead of dropping it silently. Existing - * content stays last-known-good unless the connector marks the skip authoritative. - * - `drop`: empty, non-deferred content that cannot be indexed. - * - `add` / `update` / `unchanged`: normal content reconciliation by content hash. - * - A deferred listing always rehydrates an existing content-less placeholder, - * even when its listing hash is unchanged, so a prior hydration-time skip can - * recover when the source becomes indexable. - * - * `forceRehydrate` (set on a full resync of a `rehydrateOnFullSync` connector) promotes - * an otherwise-`unchanged` deferred document to `update` so its content is re-fetched — - * needed when rendered content can drift without the hash changing (e.g. Confluence - * transclusions). Non-deferred docs already carry final content from listing, so they - * are left `unchanged` (re-indexing identical content would be pointless). - */ -export function classifyExternalDoc( - extDoc: Pick< - ExternalDocument, - | 'content' - | 'sourceFile' - | 'contentDeferred' - | 'contentHash' - | 'skippedReason' - | 'skippedExistingDisposition' - >, - existing: { id: string; contentHash: string | null; storageKey?: string | null } | undefined, - forceRehydrate = false -): DocClassification { - if (extDoc.skippedReason) { - if (!existing) return { type: 'skip' } - return shouldReplaceExistingWithSkippedDocument(existing, extDoc) - ? { type: 'skip', existingId: existing.id } - : { type: 'unchanged' } - } - if (!hasIndexablePayload(extDoc) && !extDoc.contentDeferred) { - return { type: 'drop' } - } - if (!existing) { - return { type: 'add' } - } - if (existing.storageKey === null && extDoc.contentDeferred) { - return { type: 'update', existingId: existing.id } - } - if (existing.contentHash !== extDoc.contentHash) { - return { type: 'update', existingId: existing.id } - } - if (forceRehydrate && extDoc.contentDeferred) { - return { type: 'update', existingId: existing.id } - } - return { type: 'unchanged' } -} - -/** - * Merges a hydrated document over the listing stub it was fetched for. - * - * Every field the connector restates on hydration has to be carried, not just the - * content. A stub is built before the file is fetched and declares `text/plain`, - * so any field left behind keeps a value that is wrong for the bytes now attached - * — which is how a hydrated PDF ends up still claiming plain text. Storage reads - * `sourceFile.mimeType`, so that particular staleness is invisible until - * something reaches for the obvious field instead. - * - * Extracted from the hydration loop so the merge is a stated contract with a test - * rather than an inline spread that is easy to under-specify. - */ -export function mergeHydratedDocument( - stub: ExternalDocument, - hydrated: ExternalDocument, - contentHash: string -): ExternalDocument { - return { - ...stub, - title: hydrated.title || stub.title, - content: hydrated.content, - sourceFile: hydrated.sourceFile, - mimeType: hydrated.mimeType, - contentHash, - contentDeferred: false, - sourceUrl: hydrated.sourceUrl ?? stub.sourceUrl, - metadata: { ...stub.metadata, ...hydrated.metadata }, - } -} - -/** - * Merges a hydration-time skip marker onto its listing stub. - * - * A skipped hydration did not verify indexable content, so its provider-specific - * fallback hash cannot supersede the listing hash used by the next sync's change - * classification. Keeping the listing hash makes a newly persisted skip stable - * until the source metadata changes. A connector can explicitly provide - * `skippedRetryContentHash` when the skip must be retried independently of that - * metadata, such as a Notion nested block whose access changes without editing - * its parent page. - */ -export function mergeHydratedSkippedDocument( - stub: ExternalDocument, - hydrated: ExternalDocument -): ExternalDocument { - return { - ...stub, - content: '', - contentHash: hydrated.skippedRetryContentHash ?? stub.contentHash, - contentDeferred: false, - skippedReason: hydrated.skippedReason, - skippedExistingDisposition: hydrated.skippedExistingDisposition, - metadata: { ...stub.metadata, ...hydrated.metadata }, - } -} - -/** - * A listed deferred document is known to exist at listing time. A null hydration - * is therefore ambiguous provider failure, not authoritative deletion: treating - * it as a successful drop can advance an incremental watermark past a document - * that merely became inaccessible. - */ -export function requireHydratedListedDocument( - document: ExternalDocument | null, - externalId: string -): ExternalDocument { - if (!document) { - throw new Error(`Connector returned no content for listed document ${externalId}`) - } - return document -} - -/** - * Records a source update that was observed but could not be verified or - * persisted. The stored document remains last-known-good, while `docsFailed` - * prevents an incremental watermark from advancing past the consumed change. - */ -export function recordUnverifiedExistingRefresh( - result: Pick, - failedExternalIds: Set, - externalId: string -): void { - if (failedExternalIds.has(externalId)) return - failedExternalIds.add(externalId) - result.docsFailed++ -} - -/** Actual retained bytes when available, otherwise a conservative deferred estimate. */ -function estimateOpSizeBytes(op: DocOp): number { - // Skip ops load no content (just a row insert), so they do not count against the - // in-flight content budget. - if (op.type === 'skip') return 0 - if (op.extDoc.sourceFile?.bytes) return op.extDoc.sourceFile.bytes.byteLength - if (op.extDoc.content) return Buffer.byteLength(op.extDoc.content) - const size = op.extDoc.metadata?.fileSize ?? op.extDoc.metadata?.size - return typeof size === 'number' && Number.isFinite(size) && size > 0 - ? size - : DEFAULT_OP_SIZE_BYTES -} - -/** - * Splits content ops into sub-chunks bounded by both a count (maxCount) and a summed - * byte budget, so large files are hydrated/uploaded a few at a time. A single op - * larger than the budget still forms its own chunk (always >= 1 op per chunk). - */ -export function chunkOpsByByteBudget( - ops: DocOp[], - budgetBytes: number, - maxCount: number -): DocOp[][] { - const chunks: DocOp[][] = [] - let current: DocOp[] = [] - let currentBytes = 0 - for (const op of ops) { - const bytes = estimateOpSizeBytes(op) - if (current.length > 0 && (current.length >= maxCount || currentBytes + bytes > budgetBytes)) { - chunks.push(current) - current = [] - currentBytes = 0 - } - current.push(op) - currentBytes += bytes - } - if (current.length > 0) { - chunks.push(current) - } - return chunks -} - -/** - * Single-roundtrip check that this sync's targets still exist. - * - * Named for presence rather than liveness deliberately: this file uses - * "liveness" in its distributed-systems sense — a run proving it is still - * working, via {@link heartbeatSyncLock} — and reusing the word for a row - * existence check conflated two unrelated questions three lines apart. - */ -async function checkSyncTargetPresence( - connectorId: string, - knowledgeBaseId: string -): Promise<{ connectorDeleted: boolean; knowledgeBaseDeleted: boolean }> { - const rows = await db - .select({ - connectorArchivedAt: knowledgeConnector.archivedAt, - connectorDeletedAt: knowledgeConnector.deletedAt, - kbDeletedAt: knowledgeBase.deletedAt, - }) - .from(knowledgeConnector) - .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) - .where(and(eq(knowledgeConnector.id, connectorId), eq(knowledgeBase.id, knowledgeBaseId))) - .limit(1) - - if (rows.length === 0) { - return { connectorDeleted: true, knowledgeBaseDeleted: true } - } - const row = rows[0] - return { - connectorDeleted: row.connectorArchivedAt !== null || row.connectorDeletedAt !== null, - knowledgeBaseDeleted: row.kbDeletedAt !== null, - } -} - -async function isKnowledgeBaseActiveInTx( - tx: KnowledgeBaseLockingTx, - knowledgeBaseId: string -): Promise { - await tx.execute(sql`SELECT 1 FROM knowledge_base WHERE id = ${knowledgeBaseId} FOR UPDATE`) - - const rows = await tx - .select({ id: knowledgeBase.id }) - .from(knowledgeBase) - .where(and(eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt))) - .limit(1) - - return rows.length > 0 -} - function calculateNextSyncTime(syncIntervalMinutes: number): Date | null { if (syncIntervalMinutes <= 0) return null const now = Date.now() @@ -838,6 +211,21 @@ export async function completeSuccessfulSync( .for('update') if (!lockedConnector) throw new SyncCompletionOwnershipLost() + /** + * Self-healing invariant of workspace mode: a mode switch back from + * members that was interrupted, or any other drift, leaves no document + * of this connector hidden from the workspace once a sync completes. + * Inside the completion transaction, after the lock is proven held, so a + * reclaimed run cannot rewrite a connector that has since changed mode. + */ + const restoredAcls = await restoreWorkspaceDocumentAcls(tx, connectorId) + if (restoredAcls > 0) { + logger.warn('Restored workspace access on connector documents that had drifted', { + connectorId, + restoredAcls, + }) + } + const [{ count: actualDocCount }] = await tx .select({ count: sql`count(*)::int` }) .from(document) @@ -874,15 +262,17 @@ export async function completeSuccessfulSync( const [writtenConnector] = await tx .update(knowledgeConnector) - .set( - buildSyncSuccessUpdate( + .set({ + ...buildSyncSuccessUpdate( now, actualDocCount, calculateNextSyncTime(syncIntervalMinutes), reconciliationHoldNotice, result.docsFailed === 0 - ) - ) + ), + /** Restored above, under this same lock. */ + accessRewritePending: false, + }) .where(stillHoldsSyncLock(connectorId, syncLogId)) .returning({ id: knowledgeConnector.id }) if (!writtenConnector) throw new SyncCompletionOwnershipLost() @@ -895,147 +285,6 @@ export async function completeSuccessfulSync( } } -/** - * Matches the connector row only while this run still holds its sync lock. - * - * `status = 'syncing'` alone is not enough: it asserts that *a* run holds the - * lock, not that *this* run does. Once the scheduler reclaims a stale lock and - * dispatches a replacement, the replacement sets `syncing` again — so the - * original run would match, overwrite the replacement's in-flight state and the - * reclaim's bookkeeping, and then reject the replacement's own write as - * superseded. The dead run wins and the live one loses, which is worse than the - * unguarded last-write-wins it replaced. - * - * `syncLockToken` is written in the same CAS that takes the lock, so matching it - * proves the lock is still this run's. `status` is kept alongside as defence in - * depth and to cover a user pausing the connector mid-run. - * - * Guards every write a run makes to its own connector row: both terminal paths - * and the mid-run heartbeat. The failure path needs it as much as the success - * path — a reclaimed run's failure would double-increment a counter the sweep - * already advanced and overwrite its backoff with a shorter one — and reusing it - * for the heartbeat is what turns a beat into an ownership probe. - */ -export function stillHoldsSyncLock(connectorId: string, syncLockToken: string) { - return and(holdsSyncLockToken(connectorId, syncLockToken), connectorIsLive()) -} - -/** The archived/deleted half of {@link stillHoldsSyncLock}. */ -export function connectorIsLive() { - return and(isNull(knowledgeConnector.archivedAt), isNull(knowledgeConnector.deletedAt)) -} - -/** - * Ownership only: this run still holds the lock, regardless of whether the - * connector has since been archived or deleted. - * - * The heartbeat guards on this rather than on {@link stillHoldsSyncLock} so a - * connector deleted mid-sync does not read as lock loss. It would otherwise - * raise `SyncLockLostException` before `checkSyncTargetPresence` ever ran, skipping - * the leftover-document cleanup that `ConnectorDeletedException` performs and - * leaving the sync-log row `started` until the sweep mislabelled it. Deletion is - * the liveness check's verdict to reach, not the heartbeat's. - */ -export function holdsSyncLockToken(connectorId: string, syncLockToken: string) { - return and( - eq(knowledgeConnector.id, connectorId), - eq(knowledgeConnector.status, 'syncing'), - eq(knowledgeConnector.syncLockToken, syncLockToken) - ) -} - -/** - * The connector row a run writes when it takes the sync lock. - * - * `syncLockToken` is set here, in the same statement as `status`, so ownership - * and the lock are established atomically — a token written afterwards would - * leave a window where a terminal write could not identify its own run. - * - * `syncLockLeaseAt` opens the lease at the same instant. It is deliberately not - * `updatedAt`: the reaper reads the lease, and `updatedAt` moves on every - * unrelated write to the row, so a config edit on a wedged connector used to - * renew the lock it was meant to recover. - */ -/** - * The statuses a run may take the lock from. - * - * An allowlist rather than `ne(status, 'syncing')`, because the queue outlives - * the decision to sync: a connector paused or disabled *after* its run was - * queued still had a task in flight, and a bare not-syncing test let that task - * lock the row and then write its own terminal status over the pause. This CAS - * is the single point where a run decides to start, so it is where the refusal - * belongs — the dispatch-side guards cannot see a status change that happens - * after they ran. - */ -export const LOCKABLE_CONNECTOR_STATUSES = ['active', 'error', 'pending'] as const - -export function buildSyncLockAcquisition(syncLogId: string, now: Date) { - return { - status: 'syncing' as const, - syncLockToken: syncLogId, - syncLockLeaseAt: now, - updatedAt: now, - } -} - -/** - * Whether a running sync is due to refresh its lock. - * - * Time-based rather than batch-count-based: batches vary hugely in cost, so an - * every-N-batches beat would fire constantly on small documents and barely at - * all on large ones — exactly the runs that need it. - */ -export function shouldHeartbeatSyncLock( - nowMs: number, - lastBeatMs: number, - intervalMs: number = SYNC_LOCK_HEARTBEAT_INTERVAL_MS -): boolean { - return nowMs - lastBeatMs >= intervalMs -} - -/** - * Extends the connector's lock lease to prove this run is still working, so the - * scheduler's stale-lock reclaim does not treat a slow-but-live sync as dead. - * - * Writes `syncLockLeaseAt` alone and deliberately leaves `updatedAt` untouched: - * a beat says nothing about the row's contents, and the two columns had to be - * separated so an unrelated write could stop passing for a heartbeat. - * - * Guarded on the run's own lock, so it doubles as an ownership probe: a false - * return means the lock was reclaimed and this run must stop rather than keep - * writing alongside its replacement. - */ -async function writeSyncHeartbeat( - condition: ReturnType -): Promise { - const beat = await db - .update(knowledgeConnector) - .set({ syncLockLeaseAt: new Date() }) - .where(condition) - .returning({ id: knowledgeConnector.id }) - - return beat.length > 0 -} - -export async function heartbeatSyncLock( - connectorId: string, - syncLockToken: string -): Promise { - return writeSyncHeartbeat(holdsSyncLockToken(connectorId, syncLockToken)) -} - -/** - * Extends the lease only while this run owns a live connector. Destructive - * follow-up work uses this stricter probe immediately before dispatch so a run - * reclaimed after its transaction cannot enqueue alongside its replacement. - */ -export async function heartbeatLiveSyncLock( - connectorId: string, - syncLockToken: string -): Promise { - return writeSyncHeartbeat(stillHoldsSyncLock(connectorId, syncLockToken)) -} - /** Columns a terminal write may set. Both paths write a subset of the same set. */ type ConnectorTerminalUpdate = Partial @@ -1119,216 +368,17 @@ export function markSyncSuperseded(result: SyncResult): SyncResult { } /** - * Decides whether deletion reconciliation may run for a sync. - * - * Reconciliation hard-deletes every stored document absent from the listing, - * so it must only run against a complete source set: - * - never on incremental syncs (they list only changed documents) - * - never when the engine truncated pagination (`listingTruncated`) — a forced - * fullSync cannot fix truncation, so it cannot override it - * - never when a provider declares its pagination non-authoritative - * - not when a connector capped its listing (`listingCapped`), unless a forced - * fullSync deliberately overrides the cap to reconcile the capped scope - */ -export function shouldReconcileDeletions( - isIncremental: boolean | undefined, - syncContext: Record | undefined, - fullSync: boolean | undefined -): boolean { - if (isIncremental) return false - if (syncContext?.listingTruncated) return false - if (syncContext?.reconciliationUnsafe) return false - return !syncContext?.listingCapped || Boolean(fullSync) -} - -/** - * Minimum number of documents a connector must still own before an empty - * listing is treated as suspect. Below it, an empty listing is far more likely - * to be a genuinely emptied source than a broken one, the blast radius of - * reconciling is a handful of documents, and any ratio-based judgement is - * statistically meaningless. - */ -const SUSPECT_LISTING_MIN_OWNED_DOCS = 3 -/** - * Minimum owned-document count before the proportional (collapse) guard - * applies. A source can legitimately shrink hard when it is small — going from - * 8 documents to 1 is ordinary editing — so the collapse guard only engages on - * corpora large enough that a near-total disappearance in a single sync is - * implausible without an upstream fault. - */ -const SUSPECT_COLLAPSE_MIN_OWNED_DOCS = 50 -/** - * A listing covering less than this fraction of the documents the connector - * still owns is treated as suspect. Deliberately far below any plausible - * bulk edit (10% means 10,000 documents collapsing to under 1,000) so normal - * housekeeping never trips it, while the partial-outage shapes seen in the - * wild — an auth wall or an interstitial served for most of a source — do. - */ -const SUSPECT_COLLAPSE_MAX_RATIO = 0.1 - -/** - * How many listed documents count toward the suspect-listing ratio. + * The connector row a failed sync writes. * - * `seenExternalIds` is populated before the classification loop short-circuits - * user-excluded documents, so it counts them; the owned-document denominator - * does not, because excluded rows are filtered out of the live read. Comparing - * the two directly inflates the ratio and silently weakens the collapse guard — - * with 1,000 owned / 200 excluded, a source returning 90 documents stopped - * tripping `collapsed` entirely. Subtracting the excluded documents that were - * listed puts both sides back on the same population. - */ -export function countNonExcludedListed( - seenExternalIds: ReadonlySet, - excludedExternalIds: ReadonlySet -): number { - let excludedAndListed = 0 - for (const externalId of seenExternalIds) { - if (excludedExternalIds.has(externalId)) excludedAndListed++ - } - return seenExternalIds.size - excludedAndListed -} - -/** Why a listing is considered untrustworthy evidence of deletion. */ -export type SuspectListingReason = 'empty' | 'collapsed' - -/** - * A prior sync's listing, reconstructed from its sync-log counters. - * - * `trustworthy` is false when that run could have been an incremental listing: - * an incremental run that observed no changes is indistinguishable from a full - * run that observed nothing, and treating the former as corroboration would let - * a single bad listing confirm itself. - */ -export interface PreviousListingObservation { - listedCount: number - ownedCount: number - trustworthy: boolean -} - -/** - * Classifies a listing as untrustworthy evidence that documents were deleted. - * - * A connector that returns nothing (or almost nothing) while the knowledge base - * still holds a real corpus for it is far more likely to be broken than to be - * reporting a genuinely emptied source: observed causes include an HTTP 200 - * interstitial served instead of an index, and a source moved behind auth. - * Neither surfaces as an error, so the sync looks clean and the listing looks - * authoritative. - */ -export function classifySuspectListing( - listedCount: number, - ownedCount: number -): SuspectListingReason | null { - if (ownedCount < SUSPECT_LISTING_MIN_OWNED_DOCS) return null - if (listedCount === 0) return 'empty' - if ( - ownedCount >= SUSPECT_COLLAPSE_MIN_OWNED_DOCS && - listedCount < ownedCount * SUSPECT_COLLAPSE_MAX_RATIO - ) { - return 'collapsed' - } - return null -} - -/** - * Decides whether a suspect listing may still reconcile deletions. - * - * A suspect listing is only acted on after a consecutive suspect observation, so a - * consecutive sync, so a single transient upstream fault can never remove - * documents — not even reversibly, since a soft delete hides them from search - * immediately. A genuinely emptied source keeps reconciling: its second sync - * corroborates the first and tombstones everything, and a later sync — once the - * tombstoned set is again absent — completes the two-strike purge, subject to - * {@link capReconciliationDeletions}, which withholds any generation whose - * deletion count exceeds the per-sync blast-radius cap. - * - * A forced `fullSync` overrides the guard, matching its existing meaning - * elsewhere here — an explicit human request to reconcile against this listing - * right now. - */ -export function evaluateListingSafety( - listedCount: number, - ownedCount: number, - previous: PreviousListingObservation | null, - fullSync: boolean | undefined -): { reason: SuspectListingReason | null; blocked: boolean; corroborated: boolean } { - const reason = classifySuspectListing(listedCount, ownedCount) - if (!reason) return { reason: null, blocked: false, corroborated: false } - if (fullSync) return { reason, blocked: false, corroborated: false } - - const corroborated = Boolean( - previous?.trustworthy && classifySuspectListing(previous.listedCount, previous.ownedCount) - ) - return { reason, blocked: !corroborated, corroborated } -} - -/** - * Documents a reconciliation pass could actually remove. - * - * Both reads are filtered, not just the tombstoned one: the live read already - * excludes `userExcluded` rows in SQL, so filtering it again is a no-op today, - * but it keeps this count self-consistent with - * {@link partitionSyncReconciliation}, which gates deletion on the same flag for - * both lists. The result is the denominator for the deletion cap and for - * {@link classifySuspectListing}, whose numerator - * ({@link countNonExcludedListed}) ranges over the same population. - */ -export function countDeletionEligibleOwned( - existingDocs: ReconciliationDoc[], - tombstonedDocs: ReconciliationDoc[] -): number { - return ( - existingDocs.filter((d) => !d.userExcluded).length + - tombstonedDocs.filter((d) => !d.userExcluded).length - ) -} - -/** - * Operator-facing explanation of a held reconciliation pass. - * - * Stored on `knowledgeConnector.lastSyncError` because a hold is otherwise - * invisible: the sync completes normally and an operator sees an ordinary green - * run while source-removed documents stay indexed. Names the forced full sync, - * which is the documented way to apply the removals once the source is verified. - */ -export function buildReconciliationHoldNotice( - withheld: number, - cap: number, - ownedDocCount: number, - softHeld: boolean, - hardHeld: boolean -): string { - /** - * Stated per held generation. A hard-only hold withholds documents that a - * previous sync already tombstoned, so they have been invisible since then — - * telling the operator they are "still indexed" would be false. - */ - const consequence = - softHeld && hardHeld - ? 'Documents removed at the source are still indexed, and documents already pending removal were not purged.' - : softHeld - ? 'Documents removed at the source are still indexed.' - : 'Documents already pending removal were not purged; they stay hidden from search either way.' - - return ( - `Withheld ${withheld} document removal(s) — more than the ${cap} allowed per generation ` + - `in one sync of ${ownedDocCount} documents. ${consequence} ` + - 'Check the source is returning its full contents, then run a full sync to apply the removals.' - ) -} - -/** - * The connector row a failed sync writes. - * - * Extracted for the same reason as {@link buildSyncSuccessUpdate}: this is the - * path the auto-disable breaker runs through, so the threshold and the backoff - * it applies need to be assertable without standing up the whole sync. The - * in-process ladder here and the reaper's SQL ladder must agree — they are two - * writers of one policy, both sourced from - * {@link connectorFailureBackoffMinutes}. A validated provider retry delay is - * an additional lower bound, capped at the same one-day ceiling: a short hint - * cannot weaken the failure ladder, while an untrusted extreme value cannot - * pin the connector indefinitely. + * Extracted for the same reason as {@link buildSyncSuccessUpdate}: this is the + * path the auto-disable breaker runs through, so the threshold and the backoff + * it applies need to be assertable without standing up the whole sync. The + * in-process ladder here and the reaper's SQL ladder must agree — they are two + * writers of one policy, both sourced from + * {@link connectorFailureBackoffMinutes}. A validated provider retry delay is + * an additional lower bound, capped at the same one-day ceiling: a short hint + * cannot weaken the failure ladder, while an untrusted extreme value cannot + * pin the connector indefinitely. */ export function buildSyncFailureUpdate( now: Date, @@ -1446,335 +496,6 @@ export function buildSyncSuccessUpdate( } } -/** - * The document count to attribute to the previous sync when reconstructing its - * listing. - * - * `lastSyncDocCount` counts only *visible* documents, so after a pass that - * tombstoned a corpus it collapses toward 0 — and an owned count of 0 can never - * be classified as suspect, so corroboration silently became impossible and the - * two-strike purge jammed shut. Taking the larger of the recorded count and what - * the connector owns right now (tombstones included) restores the intent: the - * previous run is judged against a corpus at least as large as the one still - * present. - */ -export function resolvePreviousOwnedCount( - lastSyncDocCount: number | null | undefined, - ownedDocCount: number -): number { - return Math.max(lastSyncDocCount ?? 0, ownedDocCount) -} - -/** - * Fraction of a connector's owned documents that a single reconciliation pass - * may remove before the pass is held. - * - * {@link SUSPECT_COLLAPSE_MAX_RATIO} only questions a listing that returns under - * 10% of the corpus, which leaves every partial-outage shape between 10% and - * 100% completely unguarded: a source that serves half its documents produces a - * listing that looks perfectly healthy to every shape guard, tombstones the - * missing half, and hard-deletes it on the next pass. 25% sits well above - * ordinary housekeeping (a quarter of a corpus removed between two syncs is - * already extraordinary) and well below the outage shapes seen in the wild. - */ -const RECONCILIATION_DELETE_MAX_RATIO = 0.25 - -/** - * Deletions always permitted regardless of ratio. - * - * The ratio is meaningless on a small corpus for the same reason - * {@link SUSPECT_COLLAPSE_MIN_OWNED_DOCS} exists — removing 20 of 40 documents - * is ordinary editing — and a floor below the collapse guard's own 50-document - * threshold keeps the cap from being the binding constraint on corpora that - * guard was written to ignore. - */ -const RECONCILIATION_DELETE_MIN_ABSOLUTE = 25 - -/** Per-connector tuning for the reconciliation blast-radius cap. */ -export interface ReconciliationDeleteCapOverride { - maxRatio?: number - minAbsolute?: number -} - -/** - * Maximum number of documents one reconciliation pass may remove. - */ -export function resolveReconciliationDeleteCap( - ownedDocCount: number, - override?: ReconciliationDeleteCapOverride -): number { - const maxRatio = override?.maxRatio ?? RECONCILIATION_DELETE_MAX_RATIO - const minAbsolute = override?.minAbsolute ?? RECONCILIATION_DELETE_MIN_ABSOLUTE - return Math.max(minAbsolute, Math.floor(Math.max(ownedDocCount, 0) * maxRatio)) -} - -/** - * Caps the blast radius of one reconciliation pass. - * - * The shape guards above all reason about listings that look *broken*. Two - * confirmed data-loss paths produce listings that look perfectly healthy and so - * pass every one of them: a partial outage returning half a corpus (above the - * 10% collapse threshold), and a change to a connector's externalId derivation, - * which yields a complete, correct listing of entirely new keys — under which - * every stored document is "absent" and every listed one is new. - * - * The hold is deliberately all-or-nothing rather than a truncation to the cap: - * deleting up to the cap still destroys data, and leaves the knowledge base in a - * state no operator asked for and no later sync can reason about. For the outage - * shapes above the corpus is left intact and reconciliation resumes as soon as - * the source returns its full listing. It does NOT self-heal from a hold caused - * by genuine bulk removal: those deletions stay withheld until a `fullSync` - * applies them, which is the point — a human confirms them. - * - * The two generations are capped SEPARATELY. Soft deletes are this sync's newly - * absent documents; hard deletes are the previous generation's soft deletes, - * confirmed absent a second time and therefore already gated by this cap once. - * Summing them double-counts the older generation and, on a connector with - * steady churn, ratchets: each sync's new soft deletes plus the prior sync's - * pending hard deletes exceed the cap, the all-or-nothing hold blocks the hard - * deletes that would drain the backlog, and the backlog grows monotonically so - * the connector never reconciles again. Capping each generation against the same - * ceiling keeps the per-sync blast radius bounded without that deadlock. - * - * Note the ceiling this yields: each generation may spend the cap independently, - * so a single sync can remove up to 2x the cap — with the default ratio, about - * half the corpus, not a quarter. That is deliberate. The two generations are - * different populations: the hard deletes were already gated by this cap on the - * sync that tombstoned them, and have been invisible ever since, so confirming - * them costs no additional visible documents. The quarter-of-a-corpus figure - * describes what one sync may newly hide, which is the number that matters for a - * source that has started lying about its contents. - * - * `fullSync` bypasses the cap, matching its meaning everywhere else here — an - * explicit human request to reconcile against this listing right now, which is - * the documented escape hatch for a genuine mass deletion. - */ -export function capReconciliationDeletions( - softDeleteIds: string[], - hardDeleteIds: string[], - ownedDocCount: number, - fullSync: boolean | undefined, - override?: ReconciliationDeleteCapOverride -): { - softDeleteIds: string[] - hardDeleteIds: string[] - held: boolean - softHeld: boolean - hardHeld: boolean - withheld: number - cap: number -} { - const cap = resolveReconciliationDeleteCap(ownedDocCount, override) - const softHeld = !fullSync && softDeleteIds.length > cap - const hardHeld = !fullSync && hardDeleteIds.length > cap - - return { - softDeleteIds: softHeld ? [] : softDeleteIds, - hardDeleteIds: hardHeld ? [] : hardDeleteIds, - held: softHeld || hardHeld, - softHeld, - hardHeld, - withheld: (softHeld ? softDeleteIds.length : 0) + (hardHeld ? hardDeleteIds.length : 0), - cap, - } -} - -/** - * Reconstructs the previous completed sync's listing from its log counters. - * - * Every document the previous run listed landed in exactly one of - * added/updated/unchanged/skipped/failed, and `lastSyncDocCount` records - * how many documents the connector owned when that run finished. Documents the - * user excluded also land in `docsUnchanged`, which can only inflate the - * reconstructed listing — erring toward "the previous listing looked healthy", - * i.e. toward blocking deletions. - */ -async function loadPreviousListingObservation( - connectorId: string, - currentSyncLogId: string, - previousOwnedCount: number, - trustworthy: boolean -): Promise { - const rows = await db - .select({ - docsAdded: knowledgeConnectorSyncLog.docsAdded, - docsUpdated: knowledgeConnectorSyncLog.docsUpdated, - docsUnchanged: knowledgeConnectorSyncLog.docsUnchanged, - docsSkipped: knowledgeConnectorSyncLog.docsSkipped, - docsFailed: knowledgeConnectorSyncLog.docsFailed, - }) - .from(knowledgeConnectorSyncLog) - .where( - and( - eq(knowledgeConnectorSyncLog.connectorId, connectorId), - eq(knowledgeConnectorSyncLog.status, 'completed'), - ne(knowledgeConnectorSyncLog.id, currentSyncLogId) - ) - ) - .orderBy(desc(knowledgeConnectorSyncLog.startedAt)) - .limit(1) - - const previous = rows[0] - if (!previous) return null - - return { - listedCount: - previous.docsAdded + - previous.docsUpdated + - previous.docsUnchanged + - previous.docsSkipped + - previous.docsFailed, - ownedCount: previousOwnedCount, - trustworthy, - } -} - -/** - * Decides whether a sync should use the connector's incremental listing. - * - * A pending-removal document only surfaces in an incremental listing if its - * content changed since last sync — an unchanged-but-still-present document - * never appears in an incremental delta at all, so it could never be - * resurrected and would stay tombstoned indefinitely on a connector that runs - * incrementally from here on. `hasTombstonedDocs` forces a full listing - * whenever any pending-removal document exists for this connector, so every - * one of them gets a real resurrect-or-confirm decision on this sync. - */ -export function shouldRunIncrementalSync( - supportsIncrementalSync: boolean | undefined, - syncMode: string | null | undefined, - fullSync: boolean | undefined, - rehydrate: boolean | undefined, - hasTombstonedDocs: boolean, - lastSyncAt: string | Date | null | undefined -): boolean { - return Boolean( - supportsIncrementalSync && - syncMode !== 'full' && - !fullSync && - !hasTombstonedDocs && - !rehydrate && - lastSyncAt != null - ) -} - -/** - * A stored document's identity, as read back for reconciliation. - * - * `userExcluded` is required, not optional. Both reads project it, so the - * deletion guards in {@link partitionSyncReconciliation} enforce something on - * their own rather than restating a filter the SQL already applied — if that - * filter were ever dropped, the guard would still hold. An optional flag made - * the guard a silent no-op on any read that forgot to select it. - */ -type ReconciliationDoc = { id: string; externalId: string | null; userExcluded: boolean } - -/** - * Partitions a connector's stored documents against the current listing into - * the three reconciliation actions. - * - * A document absent from a normal (non-fullSync) listing is never purged - * immediately — an empty or shrunken listing can equally mean a transient - * source outage, and a single bad observation must never cause an - * irreversible mass deletion. It is instead marked pending-removal - * (`softDeleteIds`), and only becomes eligible for hard deletion - * (`hardDeleteIds`) once a *later* sync confirms it's still absent — i.e. it - * was already pending-removal (`tombstonedDocs`) coming into this sync. A - * document that reappears while pending-removal is resurrected - * (`resurrectIds`) regardless of `fullSync`, since presence — unlike absence — - * is trustworthy evidence even from a partial listing. A document whose - * content refresh was attempted but failed (`failedExternalIds`) is excluded - * from resurrection even though it was seen — surfacing it now would show - * known-stale pre-tombstone content; it stays tombstoned for a later sync to - * retry. - * - * A forced `fullSync` is an explicit request to reconcile right now: it skips - * the grace period and purges everything absent in one pass. - * - * A `userExcluded` document is never deletion-eligible — the user asked to keep - * the row — but it stays fully resurrection-eligible. The distinction matters: - * `userExcluded` and `enabled` gate visibility on their own in every retrieval - * path, so resurrecting one never re-indexes it; it only clears `deletedAt`. - * Withholding resurrection instead would strand the row permanently, since the - * connector-document listing and the restore mutation both require - * `deletedAt IS NULL` — leaving it invisible, unrestorable, and (by this very - * guard) undeletable. - */ -export function partitionSyncReconciliation( - existingDocs: ReconciliationDoc[], - tombstonedDocs: ReconciliationDoc[], - seenExternalIds: Set, - failedExternalIds: Set, - fullSync: boolean | undefined -): { resurrectIds: string[]; softDeleteIds: string[]; hardDeleteIds: string[] } { - const resurrectIds = tombstonedDocs - .filter( - (d) => - d.externalId && seenExternalIds.has(d.externalId) && !failedExternalIds.has(d.externalId) - ) - .map((d) => d.id) - const liveMissingIds = existingDocs - .filter((d) => d.externalId && !d.userExcluded && !seenExternalIds.has(d.externalId)) - .map((d) => d.id) - const tombstonedStillMissingIds = tombstonedDocs - .filter((d) => d.externalId && !d.userExcluded && !seenExternalIds.has(d.externalId)) - .map((d) => d.id) - - if (fullSync) { - return { - resurrectIds, - softDeleteIds: [], - hardDeleteIds: [...liveMissingIds, ...tombstonedStillMissingIds], - } - } - return { resurrectIds, softDeleteIds: liveMissingIds, hardDeleteIds: tombstonedStillMissingIds } -} - -/** - * Re-filters the three reconciliation ID lists against a fresh ownership - * snapshot taken under the connector's `FOR UPDATE` lock, dropping any - * document a concurrent "delete connector, keep documents" request already - * detached (its `connectorId` no longer matches) since the lists were first - * computed. - */ -export function filterStillOwnedReconciliationIds( - resurrectIds: string[], - softDeleteIds: string[], - hardDeleteIds: string[], - stillOwnedIds: Set -): { resurrectIds: string[]; softDeleteIds: string[]; hardDeleteIds: string[] } { - return { - resurrectIds: resurrectIds.filter((id) => stillOwnedIds.has(id)), - softDeleteIds: softDeleteIds.filter((id) => stillOwnedIds.has(id)), - hardDeleteIds: hardDeleteIds.filter((id) => stillOwnedIds.has(id)), - } -} - -/** - * Resolves tag values from connector metadata using the connector's mapTags function. - * Translates semantic keys returned by mapTags to actual DB slots using the - * tagSlotMapping stored in sourceConfig during connector creation. - */ -export function resolveTagMapping( - connectorType: string, - metadata: Record, - sourceConfig?: Record -): Partial | undefined { - const config = CONNECTOR_REGISTRY[connectorType] - if (!config?.mapTags || !metadata) return undefined - - const semanticTags = config.mapTags(metadata) - const mapping = sourceConfig?.tagSlotMapping as Record | undefined - if (!mapping || !semanticTags) return undefined - - const result: Partial = {} - for (const [semanticKey, slot] of Object.entries(mapping)) { - const value = semanticTags[semanticKey] - ;(result as Record)[slot] = value != null ? value : null - } - return result -} - /** * Resolves an access token for a connector based on its auth mode. * OAuth connectors refresh via the credential system; API key connectors @@ -1826,8 +547,8 @@ async function resolveAccessToken( * Execute a sync for a given knowledge connector. * * This is the core sync algorithm — connector-agnostic. - * It looks up the ConnectorConfig from the registry and calls its - * listDocuments/getDocument methods. + * It looks up the ConnectorConfig from the registry and runs the shared sync + * stages under the connector's content-sync lease. */ export async function executeSync( connectorId: string, @@ -1877,6 +598,19 @@ export async function executeSync( const connectorBeforeLock = connectorRows[0] + /** + * A connector that crawls per member is driven by the member engine, whose + * lease is mutually exclusive with this one. Refused before any write so a + * stale queue entry can never run a workspace-wide crawl over it. + */ + if (connectorBeforeLock.accessMode !== 'workspace') { + logger.info('Skipping sync: connector does not sync as the workspace', { + connectorId, + accessMode: connectorBeforeLock.accessMode, + }) + return { ...result, skipReason: 'connector_not_syncable' } + } + const connectorConfig = CONNECTOR_REGISTRY[connectorBeforeLock.connectorType] if (!connectorConfig) { throw new Error(`Unknown connector type: ${connectorBeforeLock.connectorType}`) @@ -1949,6 +683,7 @@ export async function executeSync( .set(buildSyncLockAcquisition(syncLogId, new Date())) .where( and( + eq(knowledgeConnector.accessMode, 'workspace'), eq(knowledgeConnector.id, connectorId), inArray(knowledgeConnector.status, LOCKABLE_CONNECTOR_STATUSES), /** @@ -2018,22 +753,7 @@ export async function executeSync( const connector = lockResult[0] const sourceConfig = connector.sourceConfig as Record const syncStartedAt = new Date() - /** Seeded at lock acquisition, which wrote `updatedAt` itself. */ - let lastHeartbeatAtMs = Date.now() - - /** - * Refreshes the lock if the interval has elapsed, and aborts the run if it has - * been reclaimed. Called at the top of every unbounded loop in this sync — the - * time gate makes each call nearly free, so placement only has to guarantee - * that no unbounded phase runs without reaching one. - */ - const beatIfDue = async (): Promise => { - if (!shouldHeartbeatSyncLock(Date.now(), lastHeartbeatAtMs)) return - if (!(await heartbeatSyncLock(connectorId, syncLogId))) { - throw new SyncLockLostException(connectorId) - } - lastHeartbeatAtMs = Date.now() - } + const lease = createContentSyncLease(connectorId, syncLogId) await db.insert(knowledgeConnectorSyncLog).values({ id: syncLogId, connectorId, @@ -2067,11 +787,13 @@ export async function executeSync( } let accessToken = await resolveAccessToken(connector, connectorConfig, credentialUserId) + /** Re-resolves the token for every OAuth call after the first, so a long run outlives a short-lived token. */ + const refreshOAuthToken = async (): Promise => { + if (connectorConfig.auth.mode === 'oauth') { + accessToken = await resolveAccessToken(connector, connectorConfig, credentialUserId) + } + } - const externalDocs: ExternalDocument[] = [] - let retainedSourcePayloadBytes = 0 - let cursor: string | undefined - let hasMore = true const syncContext: Record = { syncRunId: generateId() } // Shared cutoff for both the tombstone-retry bound below and the stuck-document @@ -2141,52 +863,21 @@ export async function executeSync( (options?.rehydrate || options?.fullSync) && connectorConfig.rehydrateOnFullSync ) - for (let pageNum = 0; hasMore && pageNum < MAX_PAGES; pageNum++) { - /** - * Listing is where a large source spends most of its wall clock — the - * batch loop below does not start until every page has been fetched — so - * without this a big listing outran the TTL and was reclaimed as a hard - * failure, which is the exact ratchet the heartbeat exists to prevent. - */ - await beatIfDue() - - if (pageNum > 0 && connectorConfig.auth.mode === 'oauth') { - accessToken = await resolveAccessToken(connector, connectorConfig, credentialUserId) - } - - const page = await connectorConfig.listDocuments( - accessToken, - sourceConfig, - cursor, - syncContext, - lastSyncAt - ) - if (page.reconciliationSafe === false) { - syncContext.reconciliationUnsafe = true - } - if (!sourcePageFitsSyncWorkingSet(externalDocs.length, page.documents.length)) { - throw new ConnectorSyncWorkingSetLimitError(connectorId, 'source listing') - } - retainedSourcePayloadBytes = addSourcePagePayloadBytes( - retainedSourcePayloadBytes, - page.documents - ) - externalDocs.push(...page.documents) - - if (page.hasMore && !page.nextCursor) { - logger.warn('Source returned hasMore=true with no cursor, stopping pagination', { - connectorId, - pageNum, - docsSoFar: externalDocs.length, - }) - break - } - - cursor = page.nextCursor - hasMore = page.hasMore - } + const listing = await runListingPass({ + connectorId, + connectorConfig, + sourceConfig, + syncContext, + lastSyncAt, + beforePage: lease.beatIfDue, + getAccessToken: async (pageNum) => { + if (pageNum > 0) await refreshOAuthToken() + return accessToken + }, + }) + const externalDocs = listing.documents - if (hasMore) { + if (!listing.exhausted) { /** * Pagination stopped before source exhaustion (MAX_PAGES or a missing * cursor), so the listing is incomplete. `listingTruncated` blocks @@ -2206,668 +897,42 @@ export async function executeSync( connectorId, }) - /** - * Loaded sequentially with a shared sentinel budget. Three concurrent - * `SELECT`s each capped independently could still materialize three times - * the intended working set before the overflow was detected. - */ - const existingDocs = await db - .select({ - id: document.id, - externalId: document.externalId, - contentHash: document.contentHash, - storageKey: document.storageKey, - /** - * Projected as well as filtered: the SQL predicate and the in-memory guard in - * partitionSyncReconciliation must both hold, so dropping either one alone cannot make - * an excluded document deletable. - */ - userExcluded: document.userExcluded, - }) - .from(document) - .where( - and( - eq(document.connectorId, connectorId), - /** - * A user's explicit "keep but don't index" choice must never make a document eligible - * for reconciliation deletion: it is deliberately never refreshed, so its absence from - * a listing says nothing. - */ - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt) - ) - ) - .limit(syncWorkingSetQueryLimit(0)) - assertSyncWorkingSetWithinLimit(connectorId, 0, existingDocs.length) + const corpus = await loadOwnedCorpus(connectorId) + const state = createSyncRunState(result) - /** - * Documents already marked pending-removal by a prior sync's reconciliation: absent from the - * source once, not yet absent twice in a row. Including them in classification lets a document - * that reappears be recognized as existing (resurrected) rather than re-added. - */ - const tombstonedDocs = await db - .select({ - id: document.id, - externalId: document.externalId, - contentHash: document.contentHash, - storageKey: document.storageKey, - deletedAt: document.deletedAt, - /** - * Gates hard deletion in partitionSyncReconciliation without gating resurrection. - */ - userExcluded: document.userExcluded, - }) - .from(document) - .where( - and( - eq(document.connectorId, connectorId), - /** - * Load both included and user-excluded tombstones. Excluded tombstones are never - * deletion-eligible, but they must remain resurrection-eligible when their source - * document reappears or the row becomes permanently invisible and unrestorable. - */ - isNull(document.archivedAt), - isNotNull(document.deletedAt) - ) - ) - .limit(syncWorkingSetQueryLimit(existingDocs.length)) - assertSyncWorkingSetWithinLimit(connectorId, existingDocs.length, tombstonedDocs.length) + const pendingOps = classifyListing({ externalDocs, corpus, forceRehydrate, state }) - /** - * Live user-excluded rows form the third disjoint population in the shared memory budget. - * User-excluded tombstones were loaded above so source presence can clear their deletion marker; - * they are added to `excludedExternalIds` below to keep hydration short-circuited. - */ - const loadedOwnedDocs = existingDocs.length + tombstonedDocs.length - const excludedDocs = await db - .select({ externalId: document.externalId }) - .from(document) - .where( - and( - eq(document.connectorId, connectorId), - eq(document.userExcluded, true), - isNull(document.archivedAt), - isNull(document.deletedAt) - ) - ) - .limit(syncWorkingSetQueryLimit(loadedOwnedDocs)) - assertSyncWorkingSetWithinLimit(connectorId, loadedOwnedDocs, excludedDocs.length) - - const excludedExternalIds = new Set( - [ - ...excludedDocs.map((doc) => doc.externalId), - ...tombstonedDocs.filter((doc) => doc.userExcluded).map((doc) => doc.externalId), - ].filter((externalId): externalId is string => Boolean(externalId)) - ) - - const priorByExternalId = new Map( - [...existingDocs, ...tombstonedDocs] - .filter((d) => d.externalId !== null) - .map((d) => [d.externalId!, d]) - ) - - const seenExternalIds = new Set() - /** - * externalIds whose content was never verified as current: a hydration - * error, a rejected write, a fulfilled-but-unusable hydration (skipped as - * oversized, or an empty re-fetch), a listing-time skippedReason - * short-circuit, or empty non-deferred content (`drop`) — all fall back to - * either keeping the stored content as last-known-good or discarding the - * listing entry outright, without ever comparing or refreshing content. - * That's fine for an already-visible document, but for a tombstoned one it - * means we still don't have confirmed-current content — so this excludes - * them from resurrection below: a tombstoned document whose refresh didn't - * actually land must stay tombstoned rather than come back visible while - * still serving stale pre-tombstone content. - */ - const failedExternalIds = new Set() - - const pendingOps: DocOp[] = [] - for (const extDoc of externalDocs) { - if (seenExternalIds.has(extDoc.externalId)) continue - seenExternalIds.add(extDoc.externalId) - - if (excludedExternalIds.has(extDoc.externalId)) { - result.docsUnchanged++ - continue - } - - const existing = priorByExternalId.get(extDoc.externalId) - const classification = classifyExternalDoc(extDoc, existing, forceRehydrate) - - switch (classification.type) { - case 'skip': - pendingOps.push({ - type: 'skip', - existingId: classification.existingId, - extDoc, - }) - break - case 'drop': - // Empty, non-deferred content is never usable. If this was a - // reappearing tombstoned document, its content was never verified as - // current — see failedExternalIds below. - if (existing) { - recordUnverifiedExistingRefresh(result, failedExternalIds, extDoc.externalId) - } - logger.info(`Skipping empty document: ${extDoc.title}`, { - externalId: extDoc.externalId, - }) - break - case 'add': - pendingOps.push({ type: 'add', extDoc }) - break - case 'update': - pendingOps.push({ type: 'update', existingId: classification.existingId, extDoc }) - break - case 'unchanged': - // A listing-time skippedReason short-circuits classification before - // the hash comparison, so this is "kept as last-known-good", not a - // verified-unchanged match — same as the deferred-hydration - // equivalent above. A genuine hash match never sets skippedReason, - // so this only fires for the short-circuited case. - if (extDoc.skippedReason && existing) { - recordUnverifiedExistingRefresh(result, failedExternalIds, extDoc.externalId) - } else { - result.docsUnchanged++ - } - break - } - } - - // Batch by both count and summed content bytes so a few large files near the - // per-file cap never hydrate/upload together and exhaust the worker heap. - const batches = chunkOpsByByteBudget(pendingOps, CONTENT_INFLIGHT_BUDGET_BYTES, SYNC_BATCH_SIZE) - for (const rawBatch of batches) { - const presence = await checkSyncTargetPresence(connectorId, connector.knowledgeBaseId) - if (presence.connectorDeleted) { - throw new ConnectorDeletedException(connectorId) - } - if (presence.knowledgeBaseDeleted) { - throw new Error(`Knowledge base ${connector.knowledgeBaseId} was deleted during sync`) - } - - // After liveness: a deleted connector must raise ConnectorDeletedException - // and run its cleanup, not be reported as a lost lock. - await beatIfDue() - - // Oversized/skipped docs become visible `failed` rows (never silent). They are - // flagged either at listing time (skip ops here) or discovered only at fetch - // time during hydration below; both are collected and persisted after hydration. - const skipOps = rawBatch.filter((op) => op.type === 'skip') - const skippedRetryHashUpdates: Array<{ - existingId: string - externalId: string - contentHash: string - }> = [] - - const contentOps = rawBatch.filter((op) => op.type !== 'skip') - const deferredOps = contentOps.filter((op) => op.extDoc.contentDeferred) - const readyOps = contentOps.filter((op) => !op.extDoc.contentDeferred) - - if (deferredOps.length > 0) { - if (connectorConfig.auth.mode === 'oauth') { - accessToken = await resolveAccessToken(connector, connectorConfig, credentialUserId) - } - - const hydrated = await Promise.allSettled( - deferredOps.map(async (op) => { - const fullDoc = requireHydratedListedDocument( - await connectorConfig.getDocument( - accessToken!, - sourceConfig, - op.extDoc.externalId, - syncContext - ), - op.extDoc.externalId - ) - // A connector may only learn a file is too large at fetch time (its - // listing has no size). Surface that as a failed row for new files; keep - // already-indexed files as last-known-good rather than downgrading them. - if (fullDoc?.skippedReason) { - if (op.type === 'add') { - skipOps.push({ - type: 'skip', - extDoc: mergeHydratedSkippedDocument(op.extDoc, fullDoc), - }) - } else if (op.type === 'update') { - const existing = priorByExternalId.get(op.extDoc.externalId) - if (existing && shouldReplaceExistingWithSkippedDocument(existing, fullDoc)) { - skipOps.push({ - type: 'skip', - existingId: op.existingId, - extDoc: mergeHydratedSkippedDocument(op.extDoc, fullDoc), - }) - } else { - if (fullDoc.skippedRetryContentHash) { - skippedRetryHashUpdates.push({ - existingId: op.existingId, - externalId: op.extDoc.externalId, - contentHash: fullDoc.skippedRetryContentHash, - }) - } - /** Preserve last-known-good content and replay the unverified source change. */ - recordUnverifiedExistingRefresh(result, failedExternalIds, op.extDoc.externalId) - } - } - return null - } - if (!hasIndexablePayload(fullDoc)) { - /** An empty refresh cannot replace or advance past last-known-good content. */ - if (op.type === 'update') { - recordUnverifiedExistingRefresh(result, failedExternalIds, op.extDoc.externalId) - } - return null - } - const hydratedHash = fullDoc.contentHash ?? op.extDoc.contentHash - /** - * Normally an update whose hydrated hash matches the stored hash is a - * no-op (content unchanged). On a forced re-hydration the hash is - * version-based and cannot reflect the rendered-dependency change we are - * refreshing for, so re-index unconditionally instead of skipping. - */ - if ( - op.type === 'update' && - !forceRehydrate && - priorByExternalId.get(op.extDoc.externalId)?.contentHash === hydratedHash - ) { - result.docsUnchanged++ - return null - } - return { ...op, extDoc: mergeHydratedDocument(op.extDoc, fullDoc, hydratedHash) } - }) - ) - - const rateLimitFailure = hydrated.find( - (outcome): outcome is PromiseRejectedResult => - outcome.status === 'rejected' && isRateLimitError(outcome.reason) - ) - if (rateLimitFailure) { - throw rateLimitFailure.reason - } - - for (let i = 0; i < hydrated.length; i++) { - const outcome = hydrated[i] - if (outcome.status === 'fulfilled' && outcome.value) { - readyOps.push(outcome.value) - } else if (outcome.status === 'rejected') { - result.docsFailed++ - failedExternalIds.add(deferredOps[i].extDoc.externalId) - logger.error('Failed to hydrate deferred document', { - connectorId, - externalId: deferredOps[i].extDoc.externalId, - error: getErrorMessage(outcome.reason), - }) - } - } - } - - if (skippedRetryHashUpdates.length > 0) { - try { - const missedExternalIds = await persistSkippedRetryHashes( - connector.knowledgeBaseId, - connectorId, - skippedRetryHashUpdates - ) - if (missedExternalIds.length > 0) { - logger.warn('Skipped retry hashes were not persisted for detached documents', { - connectorId, - externalIds: missedExternalIds, - }) - } - } catch (error) { - logger.error('Failed to persist skipped document retry hashes', { - connectorId, - count: skippedRetryHashUpdates.length, - error: toError(error).message, - }) - throw error - } - } - - // Record all skipped (oversized) docs in this batch in one bulk insert. - if (skipOps.length > 0) { - try { - const recorded = await persistSkippedDocuments( - connector.knowledgeBaseId, - connectorId, - connector.connectorType, - skipOps, - sourceConfig - ) - result.docsSkipped += recorded - } catch (error) { - /** - * The source items were intentionally skipped, but failing to persist their visible - * failed rows is an actual sync failure. - */ - result.docsFailed += skipOps.length - for (const op of skipOps) { - failedExternalIds.add(op.extDoc.externalId) - } - logger.error('Failed to record skipped documents', { - connectorId, - count: skipOps.length, - error: toError(error).message, - }) - } - } - - const batch = readyOps - - const settled = await Promise.allSettled( - batch.map((op) => { - if (op.type === 'add') { - return addDocument( - connector.knowledgeBaseId, - connectorId, - connector.connectorType, - op.extDoc, - kbOwner, - sourceConfig - ) - } - return updateDocument( - op.existingId, - connector.knowledgeBaseId, - connectorId, - connector.connectorType, - op.extDoc, - kbOwner, - sourceConfig - ) - }) - ) - - const batchDocs: DocumentData[] = [] - for (let j = 0; j < settled.length; j++) { - const outcome = settled[j] - if (outcome.status === 'fulfilled') { - batchDocs.push(outcome.value) - if (batch[j].type === 'add') result.docsAdded++ - else result.docsUpdated++ - } else { - result.docsFailed++ - failedExternalIds.add(batch[j].extDoc.externalId) - logger.error('Failed to process document', { - connectorId, - externalId: batch[j].extDoc.externalId, - error: getErrorMessage(outcome.reason), - }) - } - } - - if (batchDocs.length > 0) { - result.processingDispatch.requested += batchDocs.length - try { - const dispatch = await processDocumentsWithQueue( - batchDocs, - connector.knowledgeBaseId, - {}, - generateId(), - billingAttribution - ) - result.processingDispatch.accepted += dispatch.accepted - result.processingDispatch.failed += dispatch.failed - } catch (error) { - result.processingDispatch.failed += batchDocs.length - logger.warn('Failed to enqueue batch for processing — will retry on next sync', { - connectorId, - count: batchDocs.length, - error: toError(error).message, - }) - } - } - } - - const { resurrectIds, softDeleteIds, hardDeleteIds } = partitionSyncReconciliation( - existingDocs, - tombstonedDocs, - seenExternalIds, - failedExternalIds, - options?.fullSync - ) + await processDocOps({ + connectorId, + connector, + sourceConfig, + kbOwner, + billingAttribution, + pendingOps, + corpus, + forceRehydrate, + state, + hydration: { + beforeHydration: refreshOAuthToken, + getDocument: (externalId) => + connectorConfig.getDocument(accessToken, sourceConfig, externalId, syncContext), + }, + lease, + documentAccess: 'workspace', + }) - let reconcileDeletionsAllowed = shouldReconcileDeletions( - isIncremental, + const reconciliationHoldNotice = await reconcileDeletions({ + connectorId, + connector, + connectorConfig, + syncLogId, syncContext, - options?.fullSync - ) - - /** - * Backstop shared by every connector: a listing that reports (almost) - * nothing while this connector still owns a real corpus is treated as a - * fault, not as evidence of deletion, until a consecutive sync sees the - * same thing. Only evaluated when reconciliation would otherwise run, so - * healthy syncs pay nothing and no existing gate is loosened. - */ - /** - * Counted over deletion-eligible rows on both sides. The live read filters - * excluded documents in SQL; the tombstoned read only projects the flag, so - * excluded tombstones must be dropped here or they inflate a denominator - * governing a population they are not part of. Matches `listedDocCount`, - * which `countNonExcludedListed` already puts on the same footing. - */ - const ownedDocCount = countDeletionEligibleOwned(existingDocs, tombstonedDocs) - /** - * Counted over the same population as `ownedDocCount`: excluded documents - * are absent from the live read, so they must not inflate the numerator. - */ - const listedDocCount = countNonExcludedListed(seenExternalIds, excludedExternalIds) - if (reconcileDeletionsAllowed && classifySuspectListing(listedDocCount, ownedDocCount)) { - const previousObservation = await loadPreviousListingObservation( - connectorId, - syncLogId, - resolvePreviousOwnedCount(connector.lastSyncDocCount, ownedDocCount), - !connectorConfig.supportsIncrementalSync || connector.syncMode === 'full' - ) - const listingSafety = evaluateListingSafety( - listedDocCount, - ownedDocCount, - previousObservation, - options?.fullSync - ) - logger.warn('Suspect connector listing detected', { - connectorId, - connectorType: connector.connectorType, - reason: listingSafety.reason, - listedDocs: listedDocCount, - listedDocsIncludingExcluded: seenExternalIds.size, - ownedDocs: ownedDocCount, - liveDocs: existingDocs.length, - tombstonedDocs: tombstonedDocs.length, - previousListedDocs: previousObservation?.listedCount ?? null, - previousObservationTrusted: previousObservation?.trustworthy ?? false, - deletionReconciliation: listingSafety.blocked ? 'skipped' : 'proceeding', - syncRunId: syncContext.syncRunId, - }) - if (listingSafety.blocked) { - reconcileDeletionsAllowed = false - } - } - - /** - * Last word after every shape guard: even a listing that looks entirely - * healthy may not remove an implausible share of the corpus in one pass. - * Applied here so it covers both the soft-delete UPDATE and the - * `hardDeleteDocuments` call below. - */ - const capped = capReconciliationDeletions( - reconcileDeletionsAllowed ? softDeleteIds : [], - reconcileDeletionsAllowed ? hardDeleteIds : [], - ownedDocCount, - options?.fullSync - ) - /** - * Surfaced on the connector so a held pass is visible to an operator rather - * than only in logs: without it the sync completes green, clears - * `lastSyncError`, and source-removed documents stay indexed with no signal. - * Written through the success update at the end of this run rather than - * here — that update sets `lastSyncError: null` unconditionally and would - * otherwise clobber this within the same sync. `status` is deliberately left - * `active`: the sync itself succeeded, and marking the connector broken - * would stop it syncing at all. - */ - let reconciliationHoldNotice: string | null = null - if (capped.held) { - reconciliationHoldNotice = buildReconciliationHoldNotice( - capped.withheld, - capped.cap, - ownedDocCount, - capped.softHeld, - capped.hardHeld - ) - logger.error('Reconciliation deletions held — exceeds per-sync blast-radius cap', { - connectorId, - connectorType: connector.connectorType, - withheld: capped.withheld, - softHeld: capped.softHeld, - hardHeld: capped.hardHeld, - requestedSoft: softDeleteIds.length, - requestedHard: hardDeleteIds.length, - cap: capped.cap, - ownedDocCount, - listedCount: listedDocCount, - syncRunId: syncContext.syncRunId, - }) - } - - const gatedSoftDeleteIds = capped.softDeleteIds - const gatedHardDeleteIds = capped.hardDeleteIds - - const candidateIds = [ - ...new Set([...resurrectIds, ...gatedSoftDeleteIds, ...gatedHardDeleteIds]), - ] - - let safeResurrectIds: string[] = [] - let safeSoftDeleteIds: string[] = [] - let safeHardDeleteIds: string[] = [] - - if (candidateIds.length > 0) { - /** - * A concurrent "delete connector, keep documents" request detaches these - * same documents (connectorId set to NULL) under the same FOR UPDATE lock - * the DELETE route takes on this connector row. Taking that lock here - * serializes the two requests: whichever commits first wins, and the - * loser's re-check below sees the up-to-date connectorId and skips any - * document the other request already claimed — instead of resurrecting or - * deleting a document that another request just detached (and possibly - * already billed) as a standalone KB entry. - */ - await db.transaction(async (tx) => { - const [activeKnowledgeBase] = await tx - .select({ id: knowledgeBase.id }) - .from(knowledgeBase) - .where( - and(eq(knowledgeBase.id, connector.knowledgeBaseId), isNull(knowledgeBase.deletedAt)) - ) - .for('update') - if (!activeKnowledgeBase) throw new SyncLockLostException(connectorId) - - const [heldSyncLock] = await tx - .select({ id: knowledgeConnector.id }) - .from(knowledgeConnector) - .where(stillHoldsSyncLock(connectorId, syncLogId)) - .for('update') - if (!heldSyncLock) throw new SyncLockLostException(connectorId) - - const stillOwned = new Set( - ( - await tx - .select({ id: document.id }) - .from(document) - .where( - and( - inArray(document.id, candidateIds), - eq(document.connectorId, connectorId), - isNull(document.archivedAt) - ) - ) - ).map((d) => d.id) - ) - - const stillOwnedResult = filterStillOwnedReconciliationIds( - resurrectIds, - gatedSoftDeleteIds, - gatedHardDeleteIds, - stillOwned - ) - safeResurrectIds = stillOwnedResult.resurrectIds - safeSoftDeleteIds = stillOwnedResult.softDeleteIds - safeHardDeleteIds = stillOwnedResult.hardDeleteIds - - /** - * A document reappearing at the source is trustworthy evidence on its - * own — unlike absence, presence never depends on the listing being - * complete — so resurrection runs unconditionally, even on an - * incremental or otherwise gated sync. - */ - if (safeResurrectIds.length > 0) { - await tx - .update(document) - .set({ deletedAt: null }) - .where( - and( - inArray(document.id, safeResurrectIds), - eq(document.connectorId, connectorId), - isNull(document.archivedAt), - isNotNull(document.deletedAt) - ) - ) - } - if (safeSoftDeleteIds.length > 0) { - await tx - .update(document) - .set({ deletedAt: new Date() }) - .where( - and( - inArray(document.id, safeSoftDeleteIds), - eq(document.connectorId, connectorId), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt) - ) - ) - } - }) - } - - if (safeResurrectIds.length > 0) { - logger.info( - `Resurrected ${safeResurrectIds.length} documents that reappeared at the source`, - { - connectorId, - } - ) - } - if (safeSoftDeleteIds.length > 0) { - logger.info( - `Marked ${safeSoftDeleteIds.length} documents pending removal — absent from source, confirming on next sync`, - { connectorId } - ) - } - for (let i = 0; i < safeHardDeleteIds.length; i += HARD_DELETE_CHUNK_SIZE) { - await beatIfDue() - try { - result.docsDeleted += await hardDeleteDocuments( - safeHardDeleteIds.slice(i, i + HARD_DELETE_CHUNK_SIZE), - syncLogId, - connectorId, - connector.knowledgeBaseId, - { - connectorId, - knowledgeBaseId: connector.knowledgeBaseId, - syncLockToken: syncLogId, - } - ) - } catch (error) { - if (error instanceof ConnectorSyncDeletionGuardError) { - throw new SyncLockLostException(connectorId) - } - throw error - } - } + isIncremental, + fullSync: options?.fullSync, + corpus, + state, + lease, + }) const postBatchPresence = await checkSyncTargetPresence(connectorId, connector.knowledgeBaseId) if (postBatchPresence.connectorDeleted) { @@ -2877,258 +942,15 @@ export async function executeSync( throw new Error(`Knowledge base ${connector.knowledgeBaseId} was deleted during sync`) } - /** - * Reclaims documents this connector left unfinished: a terminated attempt, a - * dispatch that never produced a run, or a run abandoned mid-processing. - * - * The query applies each status's age rule before the candidate limit, so - * recently requeued old uploads cannot hide genuinely overdue work. The same - * rules are evaluated again after candidate rows are locked below. Skipped - * documents are content-less `failed` rows with no storage key and therefore - * remain excluded outright. - */ - const sweepEvaluatedAt = new Date() - const queuedGraceCutoff = new Date(sweepEvaluatedAt.getTime() - QUEUED_DISPATCH_GRACE_MS) - const processingStaleCutoff = new Date( - sweepEvaluatedAt.getTime() - STALE_PROCESSING_MINUTES * 60 * 1000 - ) - const sweepCandidates = await db - .select({ - id: document.id, - fileUrl: document.fileUrl, - filename: document.filename, - fileSize: document.fileSize, - mimeType: document.mimeType, - processingStatus: document.processingStatus, - processingQueuedAt: document.processingQueuedAt, - processingStartedAt: document.processingStartedAt, - processingDeferredUntil: document.processingDeferredUntil, - processingCompletedAt: document.processingCompletedAt, - uploadedAt: document.uploadedAt, - }) - .from(document) - .where( - and( - eq(document.connectorId, connectorId), - inArray(document.processingStatus, SWEEPABLE_PROCESSING_STATUSES), - or( - and( - eq(document.processingStatus, 'failed'), - sql`COALESCE(${document.processingCompletedAt}, ${document.processingQueuedAt}, ${document.uploadedAt}) < ${sql.param(queuedGraceCutoff, document.processingCompletedAt)}` - ), - and( - eq(document.processingStatus, 'pending'), - or( - and( - isNotNull(document.processingDeferredUntil), - lt(document.processingDeferredUntil, queuedGraceCutoff) - ), - and( - isNull(document.processingDeferredUntil), - sql`COALESCE(${document.processingQueuedAt}, ${document.uploadedAt}) < ${sql.param(queuedGraceCutoff, document.processingQueuedAt)}` - ) - ) - ), - and( - eq(document.processingStatus, 'processing'), - or( - isNull(document.processingStartedAt), - lt(document.processingStartedAt, processingStaleCutoff) - ) - ) - ), - // Dead letters are left alone: past the budget, re-dispatching only - // re-bills a document that has failed the same way every time. - lt(document.processingAttempts, MAX_PROCESSING_ATTEMPTS), - lt(document.uploadedAt, syncStartedAt), - gt(document.uploadedAt, retryCutoff), - eq(document.userExcluded, false), - isNotNull(document.storageKey), - isNull(document.archivedAt), - isNull(document.deletedAt) - ) - ) - .orderBy( - asc(sql`CASE - WHEN ${document.processingStatus} = 'failed' - THEN COALESCE(${document.processingCompletedAt}, ${document.processingQueuedAt}, ${document.uploadedAt}) - WHEN ${document.processingStatus} = 'pending' - THEN COALESCE(${document.processingDeferredUntil}, ${document.processingQueuedAt}, ${document.uploadedAt}) - ELSE COALESCE(${document.processingStartedAt}, ${sql.param(new Date(0), document.processingStartedAt)}) - END`), - asc(document.id) - ) - .limit(STUCK_RETRY_MAX_CANDIDATES_PER_SYNC) - const stuckDocs = sweepCandidates.filter( - (row): row is typeof row & { processingStatus: DocumentProcessingStatus } => - isDocumentProcessingStatus(row.processingStatus) - ) - - if (stuckDocs.length > 0) { - logger.info(`Retrying ${stuckDocs.length} stuck documents`, { connectorId }) - try { - const stuckDocIds = stuckDocs.map((doc) => doc.id) - let retryDocs: typeof stuckDocs = [] - - /** - * Locks the parent first to match lifecycle mutations, then proves this - * run still owns the live connector row. A bare connector lock can match - * a replacement run after this lease was reclaimed, allowing the stale - * run to reset documents and dispatch duplicate processing. - */ - await db.transaction(async (tx) => { - const [activeKnowledgeBase] = await tx - .select({ id: knowledgeBase.id }) - .from(knowledgeBase) - .where( - and(eq(knowledgeBase.id, connector.knowledgeBaseId), isNull(knowledgeBase.deletedAt)) - ) - .for('update') - if (!activeKnowledgeBase) throw new SyncLockLostException(connectorId) - - const [heldSyncLock] = await tx - .select({ id: knowledgeConnector.id }) - .from(knowledgeConnector) - .where(stillHoldsSyncLock(connectorId, syncLogId)) - .for('update') - if (!heldSyncLock) throw new SyncLockLostException(connectorId) - - const lockedCandidates = await tx - .select({ - id: document.id, - fileUrl: document.fileUrl, - filename: document.filename, - fileSize: document.fileSize, - mimeType: document.mimeType, - processingStatus: document.processingStatus, - processingQueuedAt: document.processingQueuedAt, - processingStartedAt: document.processingStartedAt, - processingDeferredUntil: document.processingDeferredUntil, - processingCompletedAt: document.processingCompletedAt, - uploadedAt: document.uploadedAt, - }) - .from(document) - .where( - and( - inArray(document.id, stuckDocIds), - eq(document.connectorId, connectorId), - inArray(document.processingStatus, SWEEPABLE_PROCESSING_STATUSES), - lt(document.processingAttempts, MAX_PROCESSING_ATTEMPTS), - eq(document.userExcluded, false), - isNotNull(document.storageKey), - isNull(document.archivedAt), - isNull(document.deletedAt) - ) - ) - .orderBy(asc(document.id)) - .for('update') - - retryDocs = selectStuckDocumentSweepCandidates( - lockedCandidates.filter( - (row): row is typeof row & { processingStatus: DocumentProcessingStatus } => - isDocumentProcessingStatus(row.processingStatus) - ), - sweepEvaluatedAt - ) - - if (retryDocs.length > 0) { - const retryDocIds = retryDocs.map((doc) => doc.id) - - const reset = await tx - .update(document) - .set({ - processingStatus: 'pending', - /** - * Invalidates the prior dispatch generation in the same write - * that reopens the row. The dispatch below installs its fresh - * generation through `markDocumentsQueued`. - */ - processingQueuedAt: null, - processingQueueToken: null, - processingStartedAt: null, - processingDeferredUntil: null, - processingCompletedAt: null, - processingError: null, - chunkCount: 0, - tokenCount: 0, - characterCount: 0, - }) - /** - * These rows were freshly revalidated and locked above. The - * lifecycle predicates remain as defence in depth; the row locks - * ensure no retry can install a newer queue generation between - * that eligibility decision and this reset. - */ - .where( - and( - inArray(document.id, retryDocIds), - eq(document.connectorId, connectorId), - inArray(document.processingStatus, SWEEPABLE_PROCESSING_STATUSES), - lt(document.processingAttempts, MAX_PROCESSING_ATTEMPTS), - eq(document.userExcluded, false), - isNotNull(document.storageKey), - isNull(document.archivedAt), - isNull(document.deletedAt) - ) - ) - .returning({ id: document.id }) - - // Embeddings are dropped only for documents this sweep actually - // reset. Deleting first would strip a pass that completed between - // the candidate SELECT and this write. - const resetIds = reset.map((row) => row.id) - if (resetIds.length > 0) { - await tx.delete(embedding).where(inArray(embedding.documentId, resetIds)) - } - const resetIdSet = new Set(resetIds) - retryDocs = retryDocs.filter((doc) => resetIdSet.has(doc.id)) - } - }) - - for (let i = 0; i < retryDocs.length; i += STUCK_RETRY_DISPATCH_CHUNK_SIZE) { - if (!(await heartbeatLiveSyncLock(connectorId, syncLogId))) { - throw new SyncLockLostException(connectorId) - } - lastHeartbeatAtMs = Date.now() - - const retryChunk = retryDocs.slice(i, i + STUCK_RETRY_DISPATCH_CHUNK_SIZE) - result.processingDispatch.requested += retryChunk.length - const dispatch = await processDocumentsWithQueue( - retryChunk.map((doc) => ({ - documentId: doc.id, - filename: doc.filename ?? 'document.txt', - fileUrl: doc.fileUrl ?? '', - fileSize: doc.fileSize ?? 0, - mimeType: doc.mimeType ?? 'text/plain', - })), - connector.knowledgeBaseId, - {}, - generateId(), - billingAttribution - ) - result.processingDispatch.accepted += dispatch.accepted - result.processingDispatch.failed += dispatch.failed - } - } catch (error) { - /** - * Kept out of the best-effort swallow below. A run that has provably - * lost its lock would otherwise be mislabelled an enqueue failure, fall - * through and publish an atomic completed outcome, which a replacement - * run could then read as corroboration of its own listing. - */ - if (error instanceof SyncLockLostException) throw error - - logger.warn('Failed to enqueue stuck documents for reprocessing', { - connectorId, - count: stuckDocs.length, - error: toError(error).message, - }) - result.processingDispatch.failed += - result.processingDispatch.requested - - result.processingDispatch.accepted - - result.processingDispatch.failed - } - } + await sweepStuckDocuments({ + connectorId, + knowledgeBaseId: connector.knowledgeBaseId, + syncStartedAt, + retryCutoff, + billingAttribution, + result, + lease, + }) const completionLanded = await completeSuccessfulSync( connectorId, @@ -3280,430 +1102,3 @@ export async function executeSync( return result } } - -/** Owning workspace + user for a knowledge base, resolved once per sync. */ -interface KnowledgeBaseOwner { - workspaceId: string | null - userId: string -} - -/** - * Build the storage `metadata` that records a trusted ownership binding for a - * synced `kb/` object. Returns `undefined` for legacy null-workspace KBs (no - * workspace-scoped ownership to bind), which `uploadFile` treats as "no binding". - */ -function kbOwnershipMetadata( - kbOwner: KnowledgeBaseOwner, - originalName: string -): { workspaceId: string; userId: string; originalName: string } | undefined { - return kbOwner.workspaceId - ? { workspaceId: kbOwner.workspaceId, userId: kbOwner.userId, originalName } - : undefined -} - -/** Builds a content-less `failed` document row for a skipped (e.g. oversized) file. */ -function buildSkippedDocumentRow( - knowledgeBaseId: string, - connectorId: string, - connectorType: string, - extDoc: ExternalDocument, - sourceConfig?: Record -) { - const reason = extDoc.skippedReason ?? 'Document was skipped during sync' - const tagValues = extDoc.metadata - ? resolveTagMapping(connectorType, extDoc.metadata, sourceConfig) - : undefined - // Connectors put the source size under either `fileSize` or `size`; accept both - // so the skipped failed row shows the real size instead of 0. - const rawSize = extDoc.metadata?.fileSize ?? extDoc.metadata?.size - const fileSize = - typeof rawSize === 'number' && Number.isFinite(rawSize) ? Math.max(0, Math.trunc(rawSize)) : 0 - - return { - id: generateId(), - knowledgeBaseId, - filename: extDoc.title, - fileUrl: '', - storageKey: null, - fileSize, - mimeType: 'text/plain', - processingStatus: 'failed', - processingError: reason, - enabled: true, - connectorId, - externalId: extDoc.externalId, - contentHash: extDoc.contentHash, - sourceUrl: extDoc.sourceUrl ?? null, - ...tagValues, - uploadedAt: new Date(), - } -} - -/** - * Records source files that were intentionally not indexed as content-less `failed` - * documents. New rows are inserted in bulk; authoritative skips replace stale rows. - * This keeps the files visible in the knowledge base UI — with `processingError` - * explaining why — instead of silently dropping them. The rows have no storage key, - * so they are excluded from the stuck-document retry sweep (nothing to reprocess). - * - * Ordinary skips on previously indexed files remain last-known-good. A connector can - * explicitly make a skip authoritative when retaining stale content would be wrong. - * - * Returns the number of rows recorded. - */ -export async function persistSkippedDocuments( - knowledgeBaseId: string, - connectorId: string, - connectorType: string, - skipOps: Array<{ - type: 'skip' - existingId?: string - extDoc: ExternalDocument - }>, - sourceConfig?: Record -): Promise { - if (skipOps.length === 0) { - return 0 - } - const inserts = skipOps - .filter((op) => !op.existingId) - .map((op) => - buildSkippedDocumentRow(knowledgeBaseId, connectorId, connectorType, op.extDoc, sourceConfig) - ) - const replacements = skipOps.filter((op): op is typeof op & { existingId: string } => - Boolean(op.existingId) - ) - const replacedFileUrls: string[] = [] - - await db.transaction(async (tx) => { - const isActive = await isKnowledgeBaseActiveInTx(tx, knowledgeBaseId) - if (!isActive) { - throw new Error(`Knowledge base ${knowledgeBaseId} is deleted`) - } - - if (inserts.length > 0) { - await tx.insert(document).values(inserts) - } - - for (const replacement of replacements) { - const skipped = buildSkippedDocumentRow( - knowledgeBaseId, - connectorId, - connectorType, - replacement.extDoc, - sourceConfig - ) - const [current] = await tx - .select({ fileUrl: document.fileUrl }) - .from(document) - .where(connectorDocumentSyncTarget(replacement.existingId, knowledgeBaseId, connectorId)) - .for('update') - if (!current) { - throw new Error(`Document ${replacement.existingId} is no longer active`) - } - const tagValues = replacement.extDoc.metadata - ? resolveTagMapping(connectorType, replacement.extDoc.metadata, sourceConfig) - : undefined - const replaced = await tx - .update(document) - .set({ - filename: skipped.filename, - fileUrl: skipped.fileUrl, - storageKey: skipped.storageKey, - fileSize: skipped.fileSize, - mimeType: skipped.mimeType, - processingStatus: skipped.processingStatus, - processingError: skipped.processingError, - processingStartedAt: null, - processingDeferredUntil: null, - processingCompletedAt: new Date(), - processingQueuedAt: null, - processingQueueToken: null, - processingAttempts: 0, - chunkCount: 0, - tokenCount: 0, - characterCount: 0, - contentHash: skipped.contentHash, - sourceUrl: skipped.sourceUrl, - uploadedAt: skipped.uploadedAt, - deletedAt: null, - ...tagValues, - }) - .where(connectorDocumentSyncTarget(replacement.existingId, knowledgeBaseId, connectorId)) - .returning({ id: document.id }) - if (replaced.length === 0) { - throw new Error(`Document ${replacement.existingId} is no longer active`) - } - if (current.fileUrl) replacedFileUrls.push(current.fileUrl) - await tx.delete(embedding).where(eq(embedding.documentId, replacement.existingId)) - } - }) - - for (const fileUrl of replacedFileUrls) { - try { - const urlPath = new URL(fileUrl, 'http://localhost').pathname - const storageKey = extractStorageKey(urlPath) - if (storageKey && storageKey !== urlPath) { - await deleteFile({ key: storageKey, context: 'knowledge-base' }) - await deleteFileMetadata(storageKey) - } - } catch (error) { - logger.warn('Failed to delete storage for an authoritatively skipped document', { - error: toError(error).message, - }) - } - } - - return skipOps.length -} - -/** - * Persists only connector-owned retry hashes for skipped refreshes of existing - * documents. Indexed content and processing state stay last-known-good while the - * hash guarantees that unchanged listing metadata still re-enters hydration. - */ -export async function persistSkippedRetryHashes( - knowledgeBaseId: string, - connectorId: string, - updates: Array<{ existingId: string; externalId: string; contentHash: string }> -): Promise { - if (updates.length === 0) return [] - - const missedExternalIds: string[] = [] - - await db.transaction(async (tx) => { - const isActive = await isKnowledgeBaseActiveInTx(tx, knowledgeBaseId) - if (!isActive) { - throw new Error(`Knowledge base ${knowledgeBaseId} is deleted`) - } - - for (const update of updates) { - const persisted = await tx - .update(document) - .set({ contentHash: update.contentHash }) - .where(connectorDocumentSyncTarget(update.existingId, knowledgeBaseId, connectorId)) - .returning({ id: document.id }) - if (persisted.length === 0) { - missedExternalIds.push(update.externalId) - } - } - }) - - return missedExternalIds -} - -/** - * Upload content to storage as a .txt file, create a document record, - * and trigger processing via the existing pipeline. - */ -async function addDocument( - knowledgeBaseId: string, - connectorId: string, - connectorType: string, - extDoc: ExternalDocument, - kbOwner: KnowledgeBaseOwner, - sourceConfig?: Record -): Promise { - const documentId = generateId() - const artifact = connectorStoredArtifact(extDoc) - const customKey = `kb/${buildStorageKeySegment(`${Date.now()}-${documentId}-`, artifact.fileName)}` - - const fileInfo = await StorageService.uploadFile({ - file: artifact.bytes, - fileName: artifact.fileName, - contentType: artifact.mimeType, - context: 'knowledge-base', - customKey, - preserveKey: true, - metadata: kbOwnershipMetadata(kbOwner, artifact.fileName), - }) - - const fileUrl = `${getInternalApiBaseUrl()}${fileInfo.path}?context=knowledge-base` - - const tagValues = extDoc.metadata - ? resolveTagMapping(connectorType, extDoc.metadata, sourceConfig) - : undefined - - try { - await db.transaction(async (tx) => { - const isActive = await isKnowledgeBaseActiveInTx(tx, knowledgeBaseId) - if (!isActive) { - throw new Error(`Knowledge base ${knowledgeBaseId} is deleted`) - } - - await tx.insert(document).values({ - id: documentId, - knowledgeBaseId, - filename: extDoc.title, - fileUrl, - storageKey: fileInfo.key, - fileSize: artifact.bytes.length, - mimeType: artifact.mimeType, - chunkCount: 0, - tokenCount: 0, - characterCount: 0, - processingStatus: 'pending', - enabled: true, - connectorId, - externalId: extDoc.externalId, - contentHash: extDoc.contentHash, - sourceUrl: extDoc.sourceUrl ?? null, - ...tagValues, - uploadedAt: new Date(), - }) - }) - } catch (error) { - const urlPath = new URL(fileUrl, 'http://localhost').pathname - const storageKey = extractStorageKey(urlPath) - if (storageKey && storageKey !== urlPath) { - await deleteFile({ key: storageKey, context: 'knowledge-base' }).catch(() => undefined) - await deleteFileMetadata(storageKey).catch(() => undefined) - } - throw error - } - - return { - documentId, - filename: artifact.fileName, - fileUrl, - fileSize: artifact.bytes.length, - mimeType: artifact.mimeType, - } -} - -/** - * Update an existing connector-sourced document with new content. - * Updates in-place to avoid unique constraint violations on (connectorId, externalId). - */ -export function connectorDocumentSyncTarget( - documentId: string, - knowledgeBaseId: string, - connectorId: string -) { - return and( - eq(document.id, documentId), - eq(document.knowledgeBaseId, knowledgeBaseId), - eq(document.connectorId, connectorId), - eq(document.userExcluded, false), - isNull(document.archivedAt) - ) -} - -async function updateDocument( - existingDocId: string, - knowledgeBaseId: string, - connectorId: string, - connectorType: string, - extDoc: ExternalDocument, - kbOwner: KnowledgeBaseOwner, - sourceConfig?: Record -): Promise { - const existingRows = await db - .select({ fileUrl: document.fileUrl }) - .from(document) - .where(connectorDocumentSyncTarget(existingDocId, knowledgeBaseId, connectorId)) - .limit(1) - const existingRow = existingRows[0] - if (!existingRow) throw new Error(`Document ${existingDocId} is no longer active`) - const oldFileUrl = existingRow.fileUrl - - const artifact = connectorStoredArtifact(extDoc) - const customKey = `kb/${buildStorageKeySegment(`${Date.now()}-${existingDocId}-`, artifact.fileName)}` - - const fileInfo = await StorageService.uploadFile({ - file: artifact.bytes, - fileName: artifact.fileName, - contentType: artifact.mimeType, - context: 'knowledge-base', - customKey, - preserveKey: true, - metadata: kbOwnershipMetadata(kbOwner, artifact.fileName), - }) - - const fileUrl = `${getInternalApiBaseUrl()}${fileInfo.path}?context=knowledge-base` - - const tagValues = extDoc.metadata - ? resolveTagMapping(connectorType, extDoc.metadata, sourceConfig) - : undefined - - try { - await db.transaction(async (tx) => { - const isActive = await isKnowledgeBaseActiveInTx(tx, knowledgeBaseId) - if (!isActive) { - throw new Error(`Knowledge base ${knowledgeBaseId} is deleted`) - } - - await tx - .update(document) - .set({ - filename: extDoc.title, - fileUrl, - storageKey: fileInfo.key, - fileSize: artifact.bytes.length, - // Re-stated on every update: a document first stored as connector-extracted - // text and later re-synced as its source file has to stop declaring - // `text/plain`, or the pipeline's OCR routing never sees it as a PDF. - mimeType: artifact.mimeType, - contentHash: extDoc.contentHash, - sourceUrl: extDoc.sourceUrl ?? null, - ...tagValues, - processingStatus: 'pending', - /** Prevents an older delayed worker from claiming newly stored content. */ - processingQueuedAt: null, - processingQueueToken: null, - processingDeferredUntil: null, - /** A new document version starts with a fresh unattended-retry budget. */ - processingAttempts: 0, - processingStartedAt: null, - processingCompletedAt: null, - processingError: null, - uploadedAt: new Date(), - // A tombstoned document reappearing with changed content is resurrected - // in the same write as its content update — otherwise reconciliation's - // separate resurrect step would clear deletedAt while this update, gated - // on deletedAt IS NULL, rejects the row and leaves stale content active. - deletedAt: null, - }) - .where(connectorDocumentSyncTarget(existingDocId, knowledgeBaseId, connectorId)) - .returning({ id: document.id }) - .then((rows) => { - if (rows.length === 0) { - throw new Error(`Document ${existingDocId} is no longer active`) - } - }) - }) - } catch (error) { - const urlPath = new URL(fileUrl, 'http://localhost').pathname - const storageKey = extractStorageKey(urlPath) - if (storageKey && storageKey !== urlPath) { - await deleteFile({ key: storageKey, context: 'knowledge-base' }).catch(() => undefined) - await deleteFileMetadata(storageKey).catch(() => undefined) - } - throw error - } - - // Clean up old storage file and its ownership binding - if (oldFileUrl) { - try { - const urlPath = new URL(oldFileUrl, 'http://localhost').pathname - const storageKey = extractStorageKey(urlPath) - if (storageKey && storageKey !== urlPath) { - await deleteFile({ key: storageKey, context: 'knowledge-base' }) - await deleteFileMetadata(storageKey) - } - } catch (error) { - logger.warn('Failed to delete old storage file', { - documentId: existingDocId, - error: toError(error).message, - }) - } - } - - return { - documentId: existingDocId, - filename: artifact.fileName, - fileUrl, - fileSize: artifact.bytes.length, - mimeType: artifact.mimeType, - } -} diff --git a/apps/sim/lib/knowledge/connectors/sync-limits.ts b/apps/sim/lib/knowledge/connectors/sync-limits.ts index 31f4ce2edab..707e85e64a7 100644 --- a/apps/sim/lib/knowledge/connectors/sync-limits.ts +++ b/apps/sim/lib/knowledge/connectors/sync-limits.ts @@ -92,3 +92,74 @@ export function connectorFailureBackoffMinutes(failures: number): number { * negligible against the work a sync does between beats. */ export const SYNC_LOCK_HEARTBEAT_INTERVAL_MS = 5 * 60 * 1000 + +/** + * Wall-clock ceiling for one members-mode run, which crawls the source once per + * member. It does not have to cover every member: the run claims members one at + * a time until {@link MEMBER_SYNC_SOFT_BUDGET_SECONDS} and re-dispatches itself + * while any remain due, so a large group drains across consecutive runs. + */ +export const MEMBER_SYNC_MAX_DURATION_SECONDS = 3600 + +/** + * When a members-mode run stops claiming new members. Leaves headroom below + * {@link MEMBER_SYNC_MAX_DURATION_SECONDS} for the member in flight to finish + * its page, write observations, and rematerialise ACLs before the platform + * kills the run. + */ +export const MEMBER_SYNC_SOFT_BUDGET_SECONDS = 2700 + +/** Reclaim TTL for a members-mode lease; the same reasoning as {@link CONNECTOR_SYNC_STALE_LOCK_TTL_MS}. */ +export const MEMBER_SYNC_STALE_LOCK_TTL_MS = MEMBER_SYNC_MAX_DURATION_SECONDS * 2 * 1000 + +/** + * Pages one member's change-feed pass may consume in one run. The feed's + * cursor is stored past every page read, so a pass the cap stops is recorded + * as incomplete — additions are kept, removals are withheld — and the next + * run continues from where it left off; a single huge feed can never + * monopolise a run or lose a change. + * + * Deliberately not applied to a listing pass, which has no cursor to resume + * from: capping it would relist the same first pages every run and never + * grant access to the documents behind them. A listing is bounded by the run + * deadline instead, and a member no run can finish alone backs off through + * `exhaustedRunAlone`. + */ +export const MEMBER_SYNC_MAX_PAGES_PER_MEMBER = 200 + +/** + * How often each member gets a full (non-incremental) listing. Only a full + * listing can grant access to a document newly shared with the member or + * remove access to one unshared, because permission changes do not move the + * source's modified timestamps. + */ +export const MEMBER_FULL_RECRAWL_MINUTES = 720 + +/** + * The full-listing cadence for a member whose connector keeps a change feed. + * The feed reports what they gain, lose, and see modified between listings, + * so the full listing is only a periodic check that the feed missed nothing. + */ +export const MEMBER_CHANGE_FEED_FULL_RECRAWL_MINUTES = 7 * 24 * 60 + +/** + * A member whose crawls have neither started nor completed for this long is + * treated as gone: their observations are removed and the documents only they + * observed go dark. Measured against the schedule, not the wall clock, so + * queue lag in a large group never triggers it. + */ +export const MEMBER_OBSERVATION_STALE_AFTER_HOURS = 24 + +/** + * How long a suspended member (credential needs re-auth, enrollment revoked, + * option disabled) keeps their observations before the row is purged. + * Suspension already removes their token from every ACL; this window exists so + * a routine re-auth restores access without re-crawling and re-hydrating. + */ +export const MEMBER_SUSPENDED_PURGE_DAYS = 30 + +/** Days a members-mode document stays tombstoned with no observer before it is hard deleted. */ +export const MEMBER_TOMBSTONE_PURGE_DAYS = 7 + +/** Hard deletes one members-mode run may perform; bounds the blast radius of a bad run. */ +export const MEMBER_PURGE_MAX_PER_RUN = 1000 diff --git a/apps/sim/lib/knowledge/connectors/sync-lock.ts b/apps/sim/lib/knowledge/connectors/sync-lock.ts new file mode 100644 index 00000000000..4cd4bb13ef6 --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/sync-lock.ts @@ -0,0 +1,296 @@ +import { db } from '@sim/db' +import { knowledgeConnector } from '@sim/db/schema' +import { and, eq, isNull } from 'drizzle-orm' +import { SYNC_LOCK_HEARTBEAT_INTERVAL_MS } from '@/lib/knowledge/connectors/sync-limits' + +/** + * Raised when a run discovers mid-flight that it no longer holds its sync lock. + * + * Stops it doing hours of further work whose terminal write would be rejected, + * and — more importantly — stops it writing documents concurrently with the + * replacement run that took the lock. + */ +export class SyncLockLostException extends Error { + constructor(connectorId: string) { + super(`Sync lock for connector ${connectorId} was reclaimed during sync`) + this.name = 'SyncLockLostException' + } +} + +/** + * Matches the connector row only while this run still holds its sync lock. + * + * `status = 'syncing'` alone is not enough: it asserts that *a* run holds the + * lock, not that *this* run does. Once the scheduler reclaims a stale lock and + * dispatches a replacement, the replacement sets `syncing` again — so the + * original run would match, overwrite the replacement's in-flight state and the + * reclaim's bookkeeping, and then reject the replacement's own write as + * superseded. The dead run wins and the live one loses, which is worse than the + * unguarded last-write-wins it replaced. + * + * `syncLockToken` is written in the same CAS that takes the lock, so matching it + * proves the lock is still this run's. `status` is kept alongside as defence in + * depth and to cover a user pausing the connector mid-run. + * + * Guards every write a run makes to its own connector row: both terminal paths + * and the mid-run heartbeat. The failure path needs it as much as the success + * path — a reclaimed run's failure would double-increment a counter the sweep + * already advanced and overwrite its backoff with a shorter one — and reusing it + * for the heartbeat is what turns a beat into an ownership probe. + */ +export function stillHoldsSyncLock(connectorId: string, syncLockToken: string) { + return and(holdsSyncLockToken(connectorId, syncLockToken), connectorIsLive()) +} + +/** The archived/deleted half of {@link stillHoldsSyncLock}. */ +export function connectorIsLive() { + return and(isNull(knowledgeConnector.archivedAt), isNull(knowledgeConnector.deletedAt)) +} + +/** + * Ownership only: this run still holds the lock, regardless of whether the + * connector has since been archived or deleted. + * + * The heartbeat guards on this rather than on {@link stillHoldsSyncLock} so a + * connector deleted mid-sync does not read as lock loss. It would otherwise + * raise `SyncLockLostException` before `checkSyncTargetPresence` ever ran, skipping + * the leftover-document cleanup that `ConnectorDeletedException` performs and + * leaving the sync-log row `started` until the sweep mislabelled it. Deletion is + * the liveness check's verdict to reach, not the heartbeat's. + */ +export function holdsSyncLockToken(connectorId: string, syncLockToken: string) { + return and( + eq(knowledgeConnector.id, connectorId), + eq(knowledgeConnector.status, 'syncing'), + eq(knowledgeConnector.syncLockToken, syncLockToken) + ) +} + +/** + * The statuses a run may take the lock from. + * + * An allowlist rather than `ne(status, 'syncing')`, because the queue outlives + * the decision to sync: a connector paused or disabled *after* its run was + * queued still had a task in flight, and a bare not-syncing test let that task + * lock the row and then write its own terminal status over the pause. This CAS + * is the single point where a run decides to start, so it is where the refusal + * belongs — the dispatch-side guards cannot see a status change that happens + * after they ran. + */ +export const LOCKABLE_CONNECTOR_STATUSES = ['active', 'error', 'pending'] as const + +/** + * The connector row a run writes when it takes the sync lock. + * + * `syncLockToken` is set here, in the same statement as `status`, so ownership + * and the lock are established atomically — a token written afterwards would + * leave a window where a terminal write could not identify its own run. + * + * `syncLockLeaseAt` opens the lease at the same instant. It is deliberately not + * `updatedAt`: the reaper reads the lease, and `updatedAt` moves on every + * unrelated write to the row, so a config edit on a wedged connector used to + * renew the lock it was meant to recover. + */ +export function buildSyncLockAcquisition(syncLogId: string, now: Date) { + return { + status: 'syncing' as const, + syncLockToken: syncLogId, + syncLockLeaseAt: now, + updatedAt: now, + } +} + +/** + * Whether a running sync is due to refresh its lock. + * + * Time-based rather than batch-count-based: batches vary hugely in cost, so an + * every-N-batches beat would fire constantly on small documents and barely at + * all on large ones — exactly the runs that need it. + */ +export function shouldHeartbeatSyncLock( + nowMs: number, + lastBeatMs: number, + intervalMs: number = SYNC_LOCK_HEARTBEAT_INTERVAL_MS +): boolean { + return nowMs - lastBeatMs >= intervalMs +} + +/** + * Extends the connector's lock lease to prove this run is still working, so the + * scheduler's stale-lock reclaim does not treat a slow-but-live sync as dead. + * + * Writes `syncLockLeaseAt` alone and deliberately leaves `updatedAt` untouched: + * a beat says nothing about the row's contents, and the two columns had to be + * separated so an unrelated write could stop passing for a heartbeat. + * + * Guarded on the run's own lock, so it doubles as an ownership probe: a false + * return means the lock was reclaimed and this run must stop rather than keep + * writing alongside its replacement. + */ +async function writeSyncHeartbeat( + condition: ReturnType +): Promise { + const beat = await db + .update(knowledgeConnector) + .set({ syncLockLeaseAt: new Date() }) + .where(condition) + .returning({ id: knowledgeConnector.id }) + + return beat.length > 0 +} + +export async function heartbeatSyncLock( + connectorId: string, + syncLockToken: string +): Promise { + return writeSyncHeartbeat(holdsSyncLockToken(connectorId, syncLockToken)) +} + +/** + * Extends the lease only while this run owns a live connector. Destructive + * follow-up work uses this stricter probe immediately before dispatch so a run + * reclaimed after its transaction cannot enqueue alongside its replacement. + */ +export async function heartbeatLiveSyncLock( + connectorId: string, + syncLockToken: string +): Promise { + return writeSyncHeartbeat(stillHoldsSyncLock(connectorId, syncLockToken)) +} + +/** + * The lease a running sync holds on its connector row, as the sync stages see + * it. The stages never build a lock predicate or a heartbeat themselves, so an + * engine that leases a different column set supplies its own implementation + * and the stages stay agnostic about which lock they run under. + */ +export interface SyncRunLease { + /** Matches the connector row only while this run still owns a live connector. */ + stillHeld: () => ReturnType + /** + * Refreshes the lease if the interval has elapsed, and aborts the run if it + * has been reclaimed. Called at the top of every unbounded loop in a sync — + * the time gate makes each call nearly free, so placement only has to + * guarantee that no unbounded phase runs without reaching one. + */ + beatIfDue: () => Promise + /** + * The stricter probe taken immediately before destructive dispatch: extends + * the lease only while the connector is still live, and aborts otherwise. + */ + beatLive: () => Promise +} + +/** The lease a document write proves before it lands, as the run that makes it holds it. */ +export type SyncWriteLease = Pick + +/** + * Proves, inside the write's own transaction, that the run still owns the + * connector. A heartbeat taken before the batch only says the lease was held + * then; the hydration and storage work between it and the row write can + * outlast the lease. The share lock keeps the scheduler's reclaim from landing + * until this write commits, and a row that no longer matches aborts the write + * instead of landing stale content over the replacement run's. + */ +export async function assertSyncLeaseHeldInTx( + tx: Pick, + connectorId: string, + lease: SyncWriteLease +): Promise { + const [held] = await tx + .select({ id: knowledgeConnector.id }) + .from(knowledgeConnector) + .where(lease.stillHeld()) + .for('share') + if (!held) throw new SyncLockLostException(connectorId) +} + +/** + * The lease of the content sync engine, held through `syncLockToken`. The + * heartbeat clock is seeded at lock acquisition, which opened `syncLockLeaseAt`. + */ +export function createContentSyncLease(connectorId: string, syncLogId: string): SyncRunLease { + let lastHeartbeatAtMs = Date.now() + return { + stillHeld: () => stillHoldsSyncLock(connectorId, syncLogId), + beatIfDue: async () => { + if (!shouldHeartbeatSyncLock(Date.now(), lastHeartbeatAtMs)) return + if (!(await heartbeatSyncLock(connectorId, syncLogId))) { + throw new SyncLockLostException(connectorId) + } + lastHeartbeatAtMs = Date.now() + }, + beatLive: async () => { + if (!(await heartbeatLiveSyncLock(connectorId, syncLogId))) { + throw new SyncLockLostException(connectorId) + } + lastHeartbeatAtMs = Date.now() + }, + } +} + +/** + * The connector statuses a members-mode run may be queued from or take its + * lease in. The same reasoning as {@link LOCKABLE_CONNECTOR_STATUSES}: the + * queue outlives the decision to sync, so a connector paused after its member + * run was dispatched still had a task in flight, and a lease CAS that ignored + * `status` let that task crawl a paused connector. `pending` is absent because + * the member lease never coexists with the content queue's entry. + */ +export const MEMBER_LOCKABLE_CONNECTOR_STATUSES = ['active', 'error'] as const + +/** + * Ownership only, for the members-mode lease: this run still holds the + * member-sync lock, whether or not the connector is still live. The member + * engine keeps its own lease columns so neither engine can ever misread the + * other's; `kc_sync_lock_exclusive_check` guarantees they never coexist. + */ +export function holdsMemberSyncLockToken(connectorId: string, memberSyncLockToken: string) { + return and( + eq(knowledgeConnector.id, connectorId), + eq(knowledgeConnector.memberSyncStatus, 'running'), + eq(knowledgeConnector.memberSyncLockToken, memberSyncLockToken) + ) +} + +/** Matches the connector row only while this members-mode run owns a live connector. */ +export function stillHoldsMemberSyncLock(connectorId: string, memberSyncLockToken: string) { + return and(holdsMemberSyncLockToken(connectorId, memberSyncLockToken), connectorIsLive()) +} + +async function writeMemberSyncHeartbeat( + condition: ReturnType +): Promise { + const beat = await db + .update(knowledgeConnector) + .set({ memberSyncLockLeaseAt: new Date() }) + .where(condition) + .returning({ id: knowledgeConnector.id }) + + return beat.length > 0 +} + +/** + * The lease of the members-mode engine, held through `memberSyncLockToken`. + * Same shape and same guarantees as {@link createContentSyncLease}, over the + * member columns. + */ +export function createMemberSyncLease(connectorId: string, runId: string): SyncRunLease { + let lastHeartbeatAtMs = Date.now() + return { + stillHeld: () => stillHoldsMemberSyncLock(connectorId, runId), + beatIfDue: async () => { + if (!shouldHeartbeatSyncLock(Date.now(), lastHeartbeatAtMs)) return + if (!(await writeMemberSyncHeartbeat(holdsMemberSyncLockToken(connectorId, runId)))) { + throw new SyncLockLostException(connectorId) + } + lastHeartbeatAtMs = Date.now() + }, + beatLive: async () => { + if (!(await writeMemberSyncHeartbeat(stillHoldsMemberSyncLock(connectorId, runId)))) { + throw new SyncLockLostException(connectorId) + } + lastHeartbeatAtMs = Date.now() + }, + } +} diff --git a/apps/sim/lib/knowledge/connectors/sync-persistence.ts b/apps/sim/lib/knowledge/connectors/sync-persistence.ts new file mode 100644 index 00000000000..7fb7f537c22 --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/sync-persistence.ts @@ -0,0 +1,630 @@ +import { db } from '@sim/db' +import { document, embedding, knowledgeBase, knowledgeConnector } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { and, eq, exists, isNull, sql } from 'drizzle-orm' +import { getInternalApiBaseUrl } from '@/lib/core/utils/urls' +import type { DbOrTx } from '@/lib/db/types' +import { textArrayLiteral } from '@/lib/knowledge/access/predicate' +import { EMPTY_ACL, WORKSPACE_ACL } from '@/lib/knowledge/access/tokens' +import { resolveSourceModifiedAt } from '@/lib/knowledge/connectors/source-modified-at' +import { assertSyncLeaseHeldInTx, type SyncWriteLease } from '@/lib/knowledge/connectors/sync-lock' +import type { DocumentData } from '@/lib/knowledge/documents/service' +import { StorageService } from '@/lib/uploads' +import { buildStorageKeySegment } from '@/lib/uploads/core/storage-key' +import { deleteFile } from '@/lib/uploads/core/storage-service' +import { deleteFileMetadata } from '@/lib/uploads/server/metadata' +import { extractStorageKey } from '@/lib/uploads/utils/file-utils' +import { CONNECTOR_REGISTRY } from '@/connectors/registry.server' +import type { DocumentTags, ExternalDocument } from '@/connectors/types' + +const logger = createLogger('ConnectorSyncPersistence') + +/** + * Who may read a document the sync writes. A workspace-mode connector's + * documents are visible to the whole workspace on insert and on every update. + * A members-mode connector's documents are born hidden and only the member + * engine's ACL materialisation, which knows who observed them, makes them + * visible; an update never touches the ACL. + */ +export type SyncDocumentAccess = 'workspace' | 'members' + +function insertedDocumentAcl(access: SyncDocumentAccess): string[] { + return [...(access === 'members' ? EMPTY_ACL : WORKSPACE_ACL)] +} + +function updatedDocumentAcl(access: SyncDocumentAccess): { acl?: string[] } { + return access === 'members' ? {} : { acl: [...WORKSPACE_ACL] } +} + +/** + * The workspace-mode invariant, applied after every successful content sync: + * a document a workspace-mode connector owns is readable by the workspace, + * whatever a mode switch or an interrupted rewrite left behind. Idempotent and + * a no-op on a healthy connector. + */ +export async function restoreWorkspaceDocumentAcls( + executor: DbOrTx, + connectorId: string +): Promise { + const workspaceAcl = textArrayLiteral(WORKSPACE_ACL) + const restored = await executor + .update(document) + .set({ acl: [...WORKSPACE_ACL] }) + .where( + and( + eq(document.connectorId, connectorId), + sql`${document.acl} <> ${workspaceAcl}`, + exists( + executor + .select({ one: sql`1` }) + .from(knowledgeConnector) + .where( + and( + eq(knowledgeConnector.id, connectorId), + eq(knowledgeConnector.accessMode, 'workspace') + ) + ) + ) + ) + ) + .returning({ id: document.id }) + return restored.length +} + +const MAX_SAFE_TITLE_LENGTH = 200 + +/** Sanitizes a document title for use in S3 storage keys. */ +function sanitizeStorageTitle(title: string): string { + return title.replace(/[^a-zA-Z0-9.-]/g, '_').slice(0, MAX_SAFE_TITLE_LENGTH) +} + +/** + * Sanitizes a source file's name for a storage key, keeping its extension. + * + * `sanitizeStorageTitle` truncates a long title outright, which for a source file + * would cut the extension off the end — and the extension is what + * `resolveStoredArtifactExtension` reads to pick a parser. Such a document would + * still parse correctly by falling back to its display name, but only by luck; + * preserving the suffix keeps the storage key authoritative for every file rather + * than for most of them. + */ +function sanitizeStorageFileName(fileName: string): string { + const dotIndex = fileName.lastIndexOf('.') + if (dotIndex <= 0) return sanitizeStorageTitle(fileName) + + const extension = sanitizeStorageTitle(fileName.slice(dotIndex)) + const base = sanitizeStorageTitle(fileName.slice(0, dotIndex)).slice( + 0, + Math.max(1, MAX_SAFE_TITLE_LENGTH - extension.length) + ) + return base + extension +} + +/** + * The bytes to store for a connector document, together with the name and type + * that describe them. + * + * The stored object must declare the format it actually holds, because + * `resolveStoredArtifactExtension` picks the parser off its storage key. A + * connector that hands over the source file keeps that file's own name and type, + * so the shared pipeline parses it exactly as an upload of the same file — which + * is what routes PDFs to OCR. A connector that extracted text itself stores + * `.txt`, since that is what the bytes now are; keeping the source extension + * there would re-parse extracted text as the original binary. + */ +function connectorStoredArtifact(extDoc: ExternalDocument): { + bytes: Buffer + fileName: string + mimeType: string +} { + if (extDoc.sourceFile) { + return { + bytes: extDoc.sourceFile.bytes, + fileName: sanitizeStorageFileName(extDoc.sourceFile.fileName), + mimeType: extDoc.sourceFile.mimeType, + } + } + return { + bytes: Buffer.from(extDoc.content, 'utf-8'), + fileName: `${sanitizeStorageTitle(extDoc.title)}.txt`, + mimeType: 'text/plain', + } +} +type KnowledgeBaseLockingTx = Pick + +async function isKnowledgeBaseActiveInTx( + tx: KnowledgeBaseLockingTx, + knowledgeBaseId: string +): Promise { + await tx.execute(sql`SELECT 1 FROM knowledge_base WHERE id = ${knowledgeBaseId} FOR UPDATE`) + + const rows = await tx + .select({ id: knowledgeBase.id }) + .from(knowledgeBase) + .where(and(eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt))) + .limit(1) + + return rows.length > 0 +} + +/** + * Resolves tag values from connector metadata using the connector's mapTags function. + * Translates semantic keys returned by mapTags to actual DB slots using the + * tagSlotMapping stored in sourceConfig during connector creation. + */ +export function resolveTagMapping( + connectorType: string, + metadata: Record, + sourceConfig?: Record +): Partial | undefined { + const config = CONNECTOR_REGISTRY[connectorType] + if (!config?.mapTags || !metadata) return undefined + + const semanticTags = config.mapTags(metadata) + const mapping = sourceConfig?.tagSlotMapping as Record | undefined + if (!mapping || !semanticTags) return undefined + + const result: Partial = {} + for (const [semanticKey, slot] of Object.entries(mapping)) { + const value = semanticTags[semanticKey] + ;(result as Record)[slot] = value != null ? value : null + } + return result +} + +/** Owning workspace + user for a knowledge base, resolved once per sync. */ +export interface KnowledgeBaseOwner { + workspaceId: string | null + userId: string +} + +/** + * Build the storage `metadata` that records a trusted ownership binding for a + * synced `kb/` object. Returns `undefined` for legacy null-workspace KBs (no + * workspace-scoped ownership to bind), which `uploadFile` treats as "no binding". + */ +function kbOwnershipMetadata( + kbOwner: KnowledgeBaseOwner, + originalName: string +): { workspaceId: string; userId: string; originalName: string } | undefined { + return kbOwner.workspaceId + ? { workspaceId: kbOwner.workspaceId, userId: kbOwner.userId, originalName } + : undefined +} + +/** Builds a content-less `failed` document row for a skipped (e.g. oversized) file. */ +function buildSkippedDocumentRow( + knowledgeBaseId: string, + connectorId: string, + connectorType: string, + extDoc: ExternalDocument, + sourceConfig: Record | undefined, + access: SyncDocumentAccess +) { + const reason = extDoc.skippedReason ?? 'Document was skipped during sync' + const tagValues = extDoc.metadata + ? resolveTagMapping(connectorType, extDoc.metadata, sourceConfig) + : undefined + // Connectors put the source size under either `fileSize` or `size`; accept both + // so the skipped failed row shows the real size instead of 0. + const rawSize = extDoc.metadata?.fileSize ?? extDoc.metadata?.size + const fileSize = + typeof rawSize === 'number' && Number.isFinite(rawSize) ? Math.max(0, Math.trunc(rawSize)) : 0 + + return { + id: generateId(), + knowledgeBaseId, + filename: extDoc.title, + fileUrl: '', + storageKey: null, + fileSize, + mimeType: 'text/plain', + processingStatus: 'failed', + processingError: reason, + enabled: true, + connectorId, + externalId: extDoc.externalId, + contentHash: extDoc.contentHash, + sourceUrl: extDoc.sourceUrl ?? null, + sourceModifiedAt: resolveSourceModifiedAt(extDoc.metadata), + acl: insertedDocumentAcl(access), + ...tagValues, + uploadedAt: new Date(), + } +} + +/** + * Records source files that were intentionally not indexed as content-less `failed` + * documents. New rows are inserted in bulk; authoritative skips replace stale rows. + * This keeps the files visible in the knowledge base UI — with `processingError` + * explaining why — instead of silently dropping them. The rows have no storage key, + * so they are excluded from the stuck-document retry sweep (nothing to reprocess). + * + * Ordinary skips on previously indexed files remain last-known-good. A connector can + * explicitly make a skip authoritative when retaining stale content would be wrong. + * + * Returns the number of rows recorded. + */ +export async function persistSkippedDocuments( + knowledgeBaseId: string, + connectorId: string, + connectorType: string, + skipOps: Array<{ + type: 'skip' + existingId?: string + extDoc: ExternalDocument + }>, + sourceConfig: Record | undefined, + access: SyncDocumentAccess, + lease: SyncWriteLease +): Promise { + if (skipOps.length === 0) { + return 0 + } + const inserts = skipOps + .filter((op) => !op.existingId) + .map((op) => + buildSkippedDocumentRow( + knowledgeBaseId, + connectorId, + connectorType, + op.extDoc, + sourceConfig, + access + ) + ) + const replacements = skipOps.filter((op): op is typeof op & { existingId: string } => + Boolean(op.existingId) + ) + const replacedFileUrls: string[] = [] + + await db.transaction(async (tx) => { + const isActive = await isKnowledgeBaseActiveInTx(tx, knowledgeBaseId) + if (!isActive) { + throw new Error(`Knowledge base ${knowledgeBaseId} is deleted`) + } + await assertSyncLeaseHeldInTx(tx, connectorId, lease) + + if (inserts.length > 0) { + await tx.insert(document).values(inserts) + } + + for (const replacement of replacements) { + const skipped = buildSkippedDocumentRow( + knowledgeBaseId, + connectorId, + connectorType, + replacement.extDoc, + sourceConfig, + access + ) + const [current] = await tx + .select({ fileUrl: document.fileUrl }) + .from(document) + .where(connectorDocumentSyncTarget(replacement.existingId, knowledgeBaseId, connectorId)) + .for('update') + if (!current) { + throw new Error(`Document ${replacement.existingId} is no longer active`) + } + const tagValues = replacement.extDoc.metadata + ? resolveTagMapping(connectorType, replacement.extDoc.metadata, sourceConfig) + : undefined + const replaced = await tx + .update(document) + .set({ + filename: skipped.filename, + fileUrl: skipped.fileUrl, + storageKey: skipped.storageKey, + fileSize: skipped.fileSize, + mimeType: skipped.mimeType, + sourceModifiedAt: skipped.sourceModifiedAt, + processingStatus: skipped.processingStatus, + processingError: skipped.processingError, + processingStartedAt: null, + processingDeferredUntil: null, + processingCompletedAt: new Date(), + processingQueuedAt: null, + processingQueueToken: null, + processingAttempts: 0, + chunkCount: 0, + tokenCount: 0, + characterCount: 0, + contentHash: skipped.contentHash, + sourceUrl: skipped.sourceUrl, + uploadedAt: skipped.uploadedAt, + deletedAt: null, + ...updatedDocumentAcl(access), + ...tagValues, + }) + .where(connectorDocumentSyncTarget(replacement.existingId, knowledgeBaseId, connectorId)) + .returning({ id: document.id }) + if (replaced.length === 0) { + throw new Error(`Document ${replacement.existingId} is no longer active`) + } + if (current.fileUrl) replacedFileUrls.push(current.fileUrl) + await tx.delete(embedding).where(eq(embedding.documentId, replacement.existingId)) + } + }) + + for (const fileUrl of replacedFileUrls) { + try { + const urlPath = new URL(fileUrl, 'http://localhost').pathname + const storageKey = extractStorageKey(urlPath) + if (storageKey && storageKey !== urlPath) { + await deleteFile({ key: storageKey, context: 'knowledge-base' }) + await deleteFileMetadata(storageKey) + } + } catch (error) { + logger.warn('Failed to delete storage for an authoritatively skipped document', { + error: toError(error).message, + }) + } + } + + return skipOps.length +} + +/** + * Persists only connector-owned retry hashes for skipped refreshes of existing + * documents. Indexed content and processing state stay last-known-good while the + * hash guarantees that unchanged listing metadata still re-enters hydration. + */ +export async function persistSkippedRetryHashes( + knowledgeBaseId: string, + connectorId: string, + updates: Array<{ existingId: string; externalId: string; contentHash: string }>, + lease: SyncWriteLease +): Promise { + if (updates.length === 0) return [] + + const missedExternalIds: string[] = [] + + await db.transaction(async (tx) => { + const isActive = await isKnowledgeBaseActiveInTx(tx, knowledgeBaseId) + if (!isActive) { + throw new Error(`Knowledge base ${knowledgeBaseId} is deleted`) + } + await assertSyncLeaseHeldInTx(tx, connectorId, lease) + + for (const update of updates) { + const persisted = await tx + .update(document) + .set({ contentHash: update.contentHash }) + .where(connectorDocumentSyncTarget(update.existingId, knowledgeBaseId, connectorId)) + .returning({ id: document.id }) + if (persisted.length === 0) { + missedExternalIds.push(update.externalId) + } + } + }) + + return missedExternalIds +} + +/** + * Stores the document's bytes (see {@link connectorStoredArtifact}) and inserts + * its `pending` row; the caller dispatches processing. + */ +export async function addDocument( + knowledgeBaseId: string, + connectorId: string, + connectorType: string, + extDoc: ExternalDocument, + kbOwner: KnowledgeBaseOwner, + sourceConfig: Record | undefined, + access: SyncDocumentAccess, + lease: SyncWriteLease +): Promise { + const documentId = generateId() + const artifact = connectorStoredArtifact(extDoc) + const customKey = `kb/${buildStorageKeySegment(`${Date.now()}-${documentId}-`, artifact.fileName)}` + + const fileInfo = await StorageService.uploadFile({ + file: artifact.bytes, + fileName: artifact.fileName, + contentType: artifact.mimeType, + context: 'knowledge-base', + customKey, + preserveKey: true, + metadata: kbOwnershipMetadata(kbOwner, artifact.fileName), + }) + + const fileUrl = `${getInternalApiBaseUrl()}${fileInfo.path}?context=knowledge-base` + + const tagValues = extDoc.metadata + ? resolveTagMapping(connectorType, extDoc.metadata, sourceConfig) + : undefined + + try { + await db.transaction(async (tx) => { + const isActive = await isKnowledgeBaseActiveInTx(tx, knowledgeBaseId) + if (!isActive) { + throw new Error(`Knowledge base ${knowledgeBaseId} is deleted`) + } + await assertSyncLeaseHeldInTx(tx, connectorId, lease) + + await tx.insert(document).values({ + id: documentId, + knowledgeBaseId, + filename: extDoc.title, + fileUrl, + storageKey: fileInfo.key, + fileSize: artifact.bytes.length, + mimeType: artifact.mimeType, + chunkCount: 0, + tokenCount: 0, + characterCount: 0, + processingStatus: 'pending', + enabled: true, + connectorId, + externalId: extDoc.externalId, + contentHash: extDoc.contentHash, + sourceUrl: extDoc.sourceUrl ?? null, + sourceModifiedAt: resolveSourceModifiedAt(extDoc.metadata), + acl: insertedDocumentAcl(access), + ...tagValues, + uploadedAt: new Date(), + }) + }) + } catch (error) { + const urlPath = new URL(fileUrl, 'http://localhost').pathname + const storageKey = extractStorageKey(urlPath) + if (storageKey && storageKey !== urlPath) { + await deleteFile({ key: storageKey, context: 'knowledge-base' }).catch(() => undefined) + await deleteFileMetadata(storageKey).catch(() => undefined) + } + throw error + } + + return { + documentId, + filename: artifact.fileName, + fileUrl, + fileSize: artifact.bytes.length, + mimeType: artifact.mimeType, + } +} + +/** The row a connector-owned document write may target: live, connector-owned, and not user-excluded. */ +export function connectorDocumentSyncTarget( + documentId: string, + knowledgeBaseId: string, + connectorId: string +) { + return and( + eq(document.id, documentId), + eq(document.knowledgeBaseId, knowledgeBaseId), + eq(document.connectorId, connectorId), + eq(document.userExcluded, false), + isNull(document.archivedAt) + ) +} + +/** + * Update an existing connector-sourced document with new content. + * Updates in-place to avoid unique constraint violations on (connectorId, externalId). + */ +export async function updateDocument( + existingDocId: string, + knowledgeBaseId: string, + connectorId: string, + connectorType: string, + extDoc: ExternalDocument, + kbOwner: KnowledgeBaseOwner, + sourceConfig: Record | undefined, + access: SyncDocumentAccess, + lease: SyncWriteLease +): Promise { + const existingRows = await db + .select({ fileUrl: document.fileUrl }) + .from(document) + .where(connectorDocumentSyncTarget(existingDocId, knowledgeBaseId, connectorId)) + .limit(1) + const existingRow = existingRows[0] + if (!existingRow) throw new Error(`Document ${existingDocId} is no longer active`) + const oldFileUrl = existingRow.fileUrl + + const artifact = connectorStoredArtifact(extDoc) + const customKey = `kb/${buildStorageKeySegment(`${Date.now()}-${existingDocId}-`, artifact.fileName)}` + + const fileInfo = await StorageService.uploadFile({ + file: artifact.bytes, + fileName: artifact.fileName, + contentType: artifact.mimeType, + context: 'knowledge-base', + customKey, + preserveKey: true, + metadata: kbOwnershipMetadata(kbOwner, artifact.fileName), + }) + + const fileUrl = `${getInternalApiBaseUrl()}${fileInfo.path}?context=knowledge-base` + + const tagValues = extDoc.metadata + ? resolveTagMapping(connectorType, extDoc.metadata, sourceConfig) + : undefined + + try { + await db.transaction(async (tx) => { + const isActive = await isKnowledgeBaseActiveInTx(tx, knowledgeBaseId) + if (!isActive) { + throw new Error(`Knowledge base ${knowledgeBaseId} is deleted`) + } + await assertSyncLeaseHeldInTx(tx, connectorId, lease) + + await tx + .update(document) + .set({ + filename: extDoc.title, + fileUrl, + storageKey: fileInfo.key, + fileSize: artifact.bytes.length, + // Re-stated on every update: a document first stored as connector-extracted + // text and later re-synced as its source file has to stop declaring + // `text/plain`, or the pipeline's OCR routing never sees it as a PDF. + mimeType: artifact.mimeType, + contentHash: extDoc.contentHash, + sourceUrl: extDoc.sourceUrl ?? null, + sourceModifiedAt: resolveSourceModifiedAt(extDoc.metadata), + ...tagValues, + processingStatus: 'pending', + /** Prevents an older delayed worker from claiming newly stored content. */ + processingQueuedAt: null, + processingQueueToken: null, + processingDeferredUntil: null, + /** A new document version starts with a fresh unattended-retry budget. */ + processingAttempts: 0, + processingStartedAt: null, + processingCompletedAt: null, + processingError: null, + uploadedAt: new Date(), + // A tombstoned document reappearing with changed content is resurrected + // in the same write as its content update — otherwise reconciliation's + // separate resurrect step would clear deletedAt while this update, gated + // on deletedAt IS NULL, rejects the row and leaves stale content active. + deletedAt: null, + ...updatedDocumentAcl(access), + }) + .where(connectorDocumentSyncTarget(existingDocId, knowledgeBaseId, connectorId)) + .returning({ id: document.id }) + .then((rows) => { + if (rows.length === 0) { + throw new Error(`Document ${existingDocId} is no longer active`) + } + }) + }) + } catch (error) { + const urlPath = new URL(fileUrl, 'http://localhost').pathname + const storageKey = extractStorageKey(urlPath) + if (storageKey && storageKey !== urlPath) { + await deleteFile({ key: storageKey, context: 'knowledge-base' }).catch(() => undefined) + await deleteFileMetadata(storageKey).catch(() => undefined) + } + throw error + } + + if (oldFileUrl) { + try { + const urlPath = new URL(oldFileUrl, 'http://localhost').pathname + const storageKey = extractStorageKey(urlPath) + if (storageKey && storageKey !== urlPath) { + await deleteFile({ key: storageKey, context: 'knowledge-base' }) + await deleteFileMetadata(storageKey) + } + } catch (error) { + logger.warn('Failed to delete old storage file', { + documentId: existingDocId, + error: toError(error).message, + }) + } + } + + return { + documentId: existingDocId, + filename: artifact.fileName, + fileUrl, + fileSize: artifact.bytes.length, + mimeType: artifact.mimeType, + } +} diff --git a/apps/sim/lib/knowledge/connectors/sync-primitives.ts b/apps/sim/lib/knowledge/connectors/sync-primitives.ts new file mode 100644 index 00000000000..f1b3938dbd1 --- /dev/null +++ b/apps/sim/lib/knowledge/connectors/sync-primitives.ts @@ -0,0 +1,2330 @@ +import { db } from '@sim/db' +import { + document, + embedding, + knowledgeBase, + knowledgeConnector, + knowledgeConnectorSyncLog, +} from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { getErrorMessage, toError } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { and, asc, desc, eq, gt, inArray, isNotNull, isNull, lt, ne, or, sql } from 'drizzle-orm' +import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' +import { env, envNumber } from '@/lib/core/config/env' +import { SyncLockLostException, type SyncRunLease } from '@/lib/knowledge/connectors/sync-lock' +import { + addDocument, + type KnowledgeBaseOwner, + persistSkippedDocuments, + persistSkippedRetryHashes, + type SyncDocumentAccess, + updateDocument, +} from '@/lib/knowledge/connectors/sync-persistence' +import { DOCUMENT_PROCESSING_STALE_THRESHOLD_MS } from '@/lib/knowledge/documents/processing-timeouts.server' +import type { DocumentData } from '@/lib/knowledge/documents/service' +import { + ConnectorSyncDeletionGuardError, + hardDeleteDocuments, + processDocumentsWithQueue, +} from '@/lib/knowledge/documents/service' +import { + type DocumentProcessingStatus, + isDocumentProcessingStatus, + MAX_PROCESSING_ATTEMPTS, + QUEUED_DISPATCH_GRACE_MS, +} from '@/lib/knowledge/documents/types' +import { isRateLimitError } from '@/lib/knowledge/documents/utils' +import type { + ConnectorConfig, + ExternalChange, + ExternalDocument, + SyncResult, +} from '@/connectors/types' +import { hasIndexablePayload } from '@/connectors/utils' + +const logger = createLogger('ConnectorSyncPrimitives') + +export class ConnectorDeletedException extends Error { + constructor(connectorId: string) { + super(`Connector ${connectorId} was deleted during sync`) + this.name = 'ConnectorDeletedException' + } +} + +const SYNC_BATCH_SIZE = 5 +/** Unknown deferred downloads run alone; actual connector files can reach this budget. */ +const DEFAULT_OP_SIZE_BYTES = 64 * 1024 * 1024 +/** + * Max summed source bytes hydrated/uploaded concurrently within a batch. Each + * in-flight file materializes as a content string plus an upload buffer, so this + * bounds peak worker memory: a few large files near the per-file cap are processed + * in smaller sub-chunks instead of all at once, while small files still process up + * to SYNC_BATCH_SIZE at a time. + */ +const CONTENT_INFLIGHT_BUDGET_BYTES = 64 * 1024 * 1024 +const MAX_PAGES = 500 +/** + * Maximum documents retained in either the source corpus or owned corpus. + * + * The engine needs the complete source identity set and the connector's complete + * owned-document set at the same time to distinguish adds from updates and to + * reconcile deletions safely. Page-count limits alone do not bound that working + * set: a connector page can contain many documents, and an incremental connector + * can accumulate a corpus much larger than its current page. The two corpora + * coexist, so the row-count peak is twice this value plus bounded + * maps and operation references. Crossing either per-corpus ceiling fails before + * document writes. + */ +export const CONNECTOR_SYNC_MAX_CORPUS_DOCUMENTS = 50_000 + +export const CONNECTOR_SYNC_MAX_SOURCE_PAYLOAD_BYTES = 256 * 1024 * 1024 + +/** + * How many stuck documents are re-dispatched per call. + * + * The retry backlog is unbounded, and on the in-process fallback path + * `processDocumentsWithQueue` parses, embeds, and indexes every document it is + * given before returning. Handing it the whole backlog made the retry a single + * await no heartbeat could interrupt; chunking gives the beat somewhere to run. + */ +const STUCK_RETRY_DISPATCH_CHUNK_SIZE = 25 + +/** + * How many stuck-document candidates one sync will consider. + * + * {@link STUCK_RETRY_DISPATCH_CHUNK_SIZE} paces the dispatch loop but does not + * bound it — the candidate query had no limit, so a connector carrying a large + * backlog dispatched the whole thing at once. One did: 2,959 documents enqueued + * in fifteen seconds onto a queue every workspace shares, at + * {@link PROCESSING_QUEUE_CONCURRENCY} concurrent runs. Nothing was + * double-billed — those documents were genuinely unindexed — but one connector + * monopolized the queue, and each dispatch mints a fresh `requestId`, so the + * Trigger.dev idempotency key differs every pass and none of it deduplicates. + * + * 200 keeps a single sync's contribution to roughly ten minutes of queue + * occupancy at the default concurrency. A backlog larger than this is not + * dropped: candidates are taken oldest-first and whatever is left stays + * eligible, so consecutive syncs drain it steadily instead of in one burst. + */ +export const STUCK_RETRY_MAX_CANDIDATES_PER_SYNC = 200 + +/** + * How many documents reconciliation hard-deletes per call. + * + * `hardDeleteDocuments` deletes storage objects, embeddings, and rows for its + * whole argument in serialized transactions, and a forced `fullSync` overriding + * a connector's listing cap can hand it tens of thousands of ids — one await + * spanning the widest gap between heartbeats in the sync, with the deletes + * themselves the slowest work in it. Chunking gives the beat somewhere to run, + * so a long purge stops looking dead to the reaper. Sized like the dispatch + * chunk above: small enough that a chunk cannot outlast the heartbeat interval, + * large enough that the per-call overhead stays negligible. + */ +const HARD_DELETE_CHUNK_SIZE = 25 + +/** + * Concurrent `knowledge-process-document` runs, shared by every workspace. + * + * Read from the same env var the task itself is configured with rather than + * restated, so the drain estimate below cannot describe a queue depth the + * deployment does not actually run. + */ +const PROCESSING_QUEUE_CONCURRENCY = envNumber(env.KB_CONFIG_CONCURRENCY_LIMIT, 20) + +export class ConnectorSyncCapacityError extends Error {} + +export class ConnectorSyncWorkingSetLimitError extends ConnectorSyncCapacityError { + constructor(connectorId: string, scope: 'source listing' | 'change feed' | 'owned corpus') { + super( + `Connector ${connectorId} ${scope} exceeds the safe per-corpus limit of ${CONNECTOR_SYNC_MAX_CORPUS_DOCUMENTS.toLocaleString()} documents. Narrow the configured source scope or set a connector document limit before syncing again.` + ) + this.name = 'ConnectorSyncWorkingSetLimitError' + } +} + +/** + * Returns a query's sentinel-inclusive limit for the remaining working-set + * budget. The extra row proves the corpus exceeded the cap without loading the + * rest of it. + */ +export function syncWorkingSetQueryLimit(rowsAlreadyLoaded: number): number { + return Math.max(CONNECTOR_SYNC_MAX_CORPUS_DOCUMENTS - rowsAlreadyLoaded, 0) + 1 +} + +export function sourcePageFitsSyncWorkingSet(rowsAlreadyLoaded: number, pageRows: number): boolean { + return rowsAlreadyLoaded + pageRows <= CONNECTOR_SYNC_MAX_CORPUS_DOCUMENTS +} + +function assertSyncWorkingSetWithinLimit( + connectorId: string, + rowsAlreadyLoaded: number, + rowsJustLoaded: number +): void { + if (rowsAlreadyLoaded + rowsJustLoaded > CONNECTOR_SYNC_MAX_CORPUS_DOCUMENTS) { + throw new ConnectorSyncWorkingSetLimitError(connectorId, 'owned corpus') + } +} + +function retainedExternalDocumentBytes(doc: ExternalDocument): number { + let bytes = Buffer.byteLength(doc.externalId) + Buffer.byteLength(doc.title) + bytes += Buffer.byteLength(doc.content ?? '') + bytes += Buffer.byteLength(doc.sourceUrl ?? '') + bytes += Buffer.byteLength(doc.contentHash ?? '') + if (doc.sourceFile?.bytes) bytes += doc.sourceFile.bytes.byteLength + try { + bytes += Buffer.byteLength(JSON.stringify(doc.metadata ?? {})) + } catch { + bytes += DEFAULT_OP_SIZE_BYTES + } + return bytes +} + +/** Fails listing before the engine retains an unbounded inline-content corpus. */ +export function addSourcePagePayloadBytes( + retainedBytes: number, + documents: ExternalDocument[] +): number { + let nextBytes = retainedBytes + for (const doc of documents) { + nextBytes += retainedExternalDocumentBytes(doc) + if (nextBytes > CONNECTOR_SYNC_MAX_SOURCE_PAYLOAD_BYTES) { + throw new ConnectorSyncCapacityError( + `Connector source listing exceeds the safe retained-payload limit of ${CONNECTOR_SYNC_MAX_SOURCE_PAYLOAD_BYTES.toLocaleString()} bytes. Use a narrower source scope or a deferred-content connector.` + ) + } + } + return nextBytes +} + +/** + * How long a document may sit in `processing` before the sweep treats its run as + * abandoned — and deletes its embeddings and re-dispatches it. + * + * DERIVED, not fixed at 45. The sweep reclaims by deleting live work, so this + * must exceed the longest a legitimate run can take. That bound is + * `KB_CONFIG_MAX_DURATION` x `KB_CONFIG_MAX_ATTEMPTS`, which an operator can + * raise: at the previous hard-coded 45, setting `KB_CONFIG_MAX_DURATION` above + * 900s silently made every long run look abandoned, so the sweep would delete + * the embeddings of documents that were still being indexed and bill a second + * pass. Deriving it keeps the invariant true at any configuration; the floor + * preserves today's value at the defaults. + */ +const STALE_PROCESSING_MINUTES = DOCUMENT_PROCESSING_STALE_THRESHOLD_MS / (60 * 1000) +export const RETRY_WINDOW_DAYS = 7 + +/** + * Processing states the stuck-document sweep may reclaim from. + * + * One constant used by BOTH the candidate SELECT and the reset UPDATE. The + * UPDATE has to re-assert what the SELECT filtered on — the ownership re-check + * between them covers `connectorId` only, so a document that completed in that + * window would otherwise be reset and have its embeddings deleted. Sharing the + * list means the two cannot drift into disagreeing about what is reclaimable. + */ +export const SWEEPABLE_PROCESSING_STATUSES = ['pending', 'failed', 'processing'] as const + +/** The processing state the stuck-document sweep decides on, one row at a time. */ +export interface StuckDocumentSweepCandidate { + processingStatus: DocumentProcessingStatus + processingQueuedAt: Date | null + processingStartedAt: Date | null + processingDeferredUntil: Date | null + processingCompletedAt: Date | null + uploadedAt: Date +} + +/** + * Decides whether the sweep may reclaim one document — delete its embeddings, + * reset it, and dispatch it again. + * + * Since document processing is dispatched to `knowledge-process-document` + * rather than awaited inline, a document sits at `pending` from dispatch until + * a worker claims it; `processing` is only written once a worker has actually + * started. Reclaiming a `pending` document therefore risks racing a run that is + * still queued, which both duplicates its work and bills a second indexing + * pass, so queued documents get {@link QUEUED_DISPATCH_GRACE_MS} before they + * are considered lost — the same grace the user-facing retry waits out before + * it will admit a `pending` document. + * + * Queue wait is measured from `processingQueuedAt`, stamped in one place — + * `markDocumentsQueued`, which every dispatch funnels through, so the column + * always describes the attempt that is live right now. + * It falls back to `uploadedAt` when NULL, which covers a document dispatched + * by the sync that created it (`uploadedAt` then sits within that sync's own + * runtime, an over-estimate bounded by the one-hour sync ceiling) and rows + * written before the column existed. + * + * `failed` is not a terminal state and gets the same grace. `processDocumentAsync` + * records the failure and then rethrows, so `knowledge-process-document` retries + * it up to `maxAttempts` (3): between attempts the row reads `failed` while a + * live run is scheduled to pick it up again. The gap between attempts is bounded + * by the queue, not by run duration — a retried run re-enters the same queue + * behind the same global concurrency limit — so `maxDuration` x `maxAttempts` + * (30 minutes) and `STALE_PROCESSING_MINUTES` are both far too short to be safe + * here: on the very backlog this grace exists for, the next attempt starts hours + * after the last one ended. `failed` is therefore aged from + * `processingCompletedAt`, the instant the last attempt ended, which every + * failure write stamps. + * + * A document whose retries genuinely exhaust is still recovered: its final + * failure stops moving `processingCompletedAt`, so one grace later it becomes + * eligible and the next sync re-dispatches it. Recovery is delayed by the grace, + * never lost. The user-facing retry stays immediate — it writes `pending` and + * dispatches without consulting the sweep at all. + * + * The grace decides when a queued run may be superseded, but correctness does + * not depend on that timing judgment. Every task carries the queue stamp its + * dispatch installed and must match it before claiming or billing the row. A + * sweep clears the abandoned stamp before installing a new one, so a late old + * task declines while the replacement proceeds. Queue admission also claims + * only an empty stamp, preventing concurrent callers from charging or enqueuing + * two live generations for the same pending document. + */ +export function isStuckDocumentSweepEligible(doc: StuckDocumentSweepCandidate, now: Date): boolean { + switch (doc.processingStatus) { + case 'failed': { + const lastAttemptEndedAt = + doc.processingCompletedAt ?? doc.processingQueuedAt ?? doc.uploadedAt + return now.getTime() - lastAttemptEndedAt.getTime() > QUEUED_DISPATCH_GRACE_MS + } + case 'pending': { + if (doc.processingDeferredUntil) { + return now.getTime() - doc.processingDeferredUntil.getTime() > QUEUED_DISPATCH_GRACE_MS + } + const queuedAt = doc.processingQueuedAt ?? doc.uploadedAt + return now.getTime() - queuedAt.getTime() > QUEUED_DISPATCH_GRACE_MS + } + case 'processing': { + if (!doc.processingStartedAt) return true + return ( + now.getTime() - doc.processingStartedAt.getTime() > STALE_PROCESSING_MINUTES * 60 * 1000 + ) + } + // No `default`: a status added to DocumentProcessingStatus must fail + // type-check here rather than silently reading as "not eligible". + case 'completed': + return false + } +} + +export function stuckDocumentSweepAgeAnchor(doc: StuckDocumentSweepCandidate): Date { + switch (doc.processingStatus) { + case 'failed': + return doc.processingCompletedAt ?? doc.processingQueuedAt ?? doc.uploadedAt + case 'pending': + return doc.processingDeferredUntil ?? doc.processingQueuedAt ?? doc.uploadedAt + case 'processing': + return doc.processingStartedAt ?? new Date(0) + case 'completed': + return doc.uploadedAt + } +} + +export function selectStuckDocumentSweepCandidates< + T extends StuckDocumentSweepCandidate & { id: string }, +>(documents: T[], now: Date, limit = STUCK_RETRY_MAX_CANDIDATES_PER_SYNC): T[] { + return documents + .filter((doc) => isStuckDocumentSweepEligible(doc, now)) + .sort((left, right) => { + const ageOrder = + stuckDocumentSweepAgeAnchor(left).getTime() - stuckDocumentSweepAgeAnchor(right).getTime() + return ageOrder || left.id.localeCompare(right.id) + }) + .slice(0, limit) +} + +export type DocOp = + | { type: 'add'; extDoc: ExternalDocument } + | { type: 'update'; existingId: string; extDoc: ExternalDocument } + | { type: 'skip'; existingId?: string; extDoc: ExternalDocument } + +type DocClassification = + | { type: 'add' } + | { type: 'update'; existingId: string } + | { type: 'skip'; existingId?: string } + | { type: 'unchanged' } + | { type: 'drop' } + +export function shouldReplaceExistingWithSkippedDocument( + existing: { storageKey?: string | null }, + skipped: Pick +): boolean { + return existing.storageKey === null || skipped.skippedExistingDisposition === 'replace' +} + +/** + * Decides what a listed external document becomes during reconciliation. + * + * - `skip`: connector flagged it (e.g. too large) and it is not already indexed — + * record a visible `failed` document instead of dropping it silently. Existing + * content stays last-known-good unless the connector marks the skip authoritative. + * - `drop`: empty, non-deferred content that cannot be indexed. + * - `add` / `update` / `unchanged`: normal content reconciliation by content hash. + * - A deferred listing always rehydrates an existing content-less placeholder, + * even when its listing hash is unchanged, so a prior hydration-time skip can + * recover when the source becomes indexable. + * + * `forceRehydrate` (set on a full resync of a `rehydrateOnFullSync` connector) promotes + * an otherwise-`unchanged` deferred document to `update` so its content is re-fetched — + * needed when rendered content can drift without the hash changing (e.g. Confluence + * transclusions). Non-deferred docs already carry final content from listing, so they + * are left `unchanged` (re-indexing identical content would be pointless). + */ +export function classifyExternalDoc( + extDoc: Pick< + ExternalDocument, + | 'content' + | 'sourceFile' + | 'contentDeferred' + | 'contentHash' + | 'skippedReason' + | 'skippedExistingDisposition' + >, + existing: { id: string; contentHash: string | null; storageKey?: string | null } | undefined, + forceRehydrate = false +): DocClassification { + if (extDoc.skippedReason) { + if (!existing) return { type: 'skip' } + return shouldReplaceExistingWithSkippedDocument(existing, extDoc) + ? { type: 'skip', existingId: existing.id } + : { type: 'unchanged' } + } + if (!hasIndexablePayload(extDoc) && !extDoc.contentDeferred) { + return { type: 'drop' } + } + if (!existing) { + return { type: 'add' } + } + if (existing.storageKey === null && extDoc.contentDeferred) { + return { type: 'update', existingId: existing.id } + } + if (existing.contentHash !== extDoc.contentHash) { + return { type: 'update', existingId: existing.id } + } + if (forceRehydrate && extDoc.contentDeferred) { + return { type: 'update', existingId: existing.id } + } + return { type: 'unchanged' } +} + +/** + * Merges a hydrated document over the listing stub it was fetched for. + * + * Every field the connector restates on hydration has to be carried, not just the + * content. A stub is built before the file is fetched and declares `text/plain`, + * so any field left behind keeps a value that is wrong for the bytes now attached + * — which is how a hydrated PDF ends up still claiming plain text. Storage reads + * `sourceFile.mimeType`, so that particular staleness is invisible until + * something reaches for the obvious field instead. + * + * Extracted from the hydration loop so the merge is a stated contract with a test + * rather than an inline spread that is easy to under-specify. + */ +export function mergeHydratedDocument( + stub: ExternalDocument, + hydrated: ExternalDocument, + contentHash: string +): ExternalDocument { + return { + ...stub, + title: hydrated.title || stub.title, + content: hydrated.content, + sourceFile: hydrated.sourceFile, + mimeType: hydrated.mimeType, + contentHash, + contentDeferred: false, + sourceUrl: hydrated.sourceUrl ?? stub.sourceUrl, + metadata: { ...stub.metadata, ...hydrated.metadata }, + } +} + +/** + * Merges a hydration-time skip marker onto its listing stub. + * + * A skipped hydration did not verify indexable content, so its provider-specific + * fallback hash cannot supersede the listing hash used by the next sync's change + * classification. Keeping the listing hash makes a newly persisted skip stable + * until the source metadata changes. A connector can explicitly provide + * `skippedRetryContentHash` when the skip must be retried independently of that + * metadata, such as a Notion nested block whose access changes without editing + * its parent page. + */ +export function mergeHydratedSkippedDocument( + stub: ExternalDocument, + hydrated: ExternalDocument +): ExternalDocument { + return { + ...stub, + content: '', + contentHash: hydrated.skippedRetryContentHash ?? stub.contentHash, + contentDeferred: false, + skippedReason: hydrated.skippedReason, + skippedExistingDisposition: hydrated.skippedExistingDisposition, + metadata: { ...stub.metadata, ...hydrated.metadata }, + } +} + +/** + * A listed deferred document is known to exist at listing time. A null hydration + * is therefore ambiguous provider failure, not authoritative deletion: treating + * it as a successful drop can advance an incremental watermark past a document + * that merely became inaccessible. + */ +export function requireHydratedListedDocument( + document: ExternalDocument | null, + externalId: string +): ExternalDocument { + if (!document) { + throw new Error(`Connector returned no content for listed document ${externalId}`) + } + return document +} + +/** + * Records a source update that was observed but could not be verified or + * persisted. The stored document remains last-known-good, while `docsFailed` + * prevents an incremental watermark from advancing past the consumed change. + */ +export function recordUnverifiedExistingRefresh( + result: Pick, + failedExternalIds: Set, + externalId: string +): void { + if (failedExternalIds.has(externalId)) return + failedExternalIds.add(externalId) + result.docsFailed++ +} + +/** Actual retained bytes when available, otherwise a conservative deferred estimate. */ +function estimateOpSizeBytes(op: DocOp): number { + // Skip ops load no content (just a row insert), so they do not count against the + // in-flight content budget. + if (op.type === 'skip') return 0 + if (op.extDoc.sourceFile?.bytes) return op.extDoc.sourceFile.bytes.byteLength + if (op.extDoc.content) return Buffer.byteLength(op.extDoc.content) + const size = op.extDoc.metadata?.fileSize ?? op.extDoc.metadata?.size + return typeof size === 'number' && Number.isFinite(size) && size > 0 + ? size + : DEFAULT_OP_SIZE_BYTES +} + +/** + * Splits content ops into sub-chunks bounded by both a count (maxCount) and a summed + * byte budget, so large files are hydrated/uploaded a few at a time. A single op + * larger than the budget still forms its own chunk (always >= 1 op per chunk). + */ +export function chunkOpsByByteBudget( + ops: DocOp[], + budgetBytes: number, + maxCount: number +): DocOp[][] { + const chunks: DocOp[][] = [] + let current: DocOp[] = [] + let currentBytes = 0 + for (const op of ops) { + const bytes = estimateOpSizeBytes(op) + if (current.length > 0 && (current.length >= maxCount || currentBytes + bytes > budgetBytes)) { + chunks.push(current) + current = [] + currentBytes = 0 + } + current.push(op) + currentBytes += bytes + } + if (current.length > 0) { + chunks.push(current) + } + return chunks +} + +/** + * Single-roundtrip check that this sync's targets still exist. + * + * Named for presence rather than liveness deliberately: this file uses + * "liveness" in its distributed-systems sense — a run proving it is still + * working, via {@link heartbeatSyncLock} — and reusing the word for a row + * existence check conflated two unrelated questions three lines apart. + */ +export async function checkSyncTargetPresence( + connectorId: string, + knowledgeBaseId: string +): Promise<{ connectorDeleted: boolean; knowledgeBaseDeleted: boolean }> { + const rows = await db + .select({ + connectorArchivedAt: knowledgeConnector.archivedAt, + connectorDeletedAt: knowledgeConnector.deletedAt, + kbDeletedAt: knowledgeBase.deletedAt, + }) + .from(knowledgeConnector) + .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) + .where(and(eq(knowledgeConnector.id, connectorId), eq(knowledgeBase.id, knowledgeBaseId))) + .limit(1) + + if (rows.length === 0) { + return { connectorDeleted: true, knowledgeBaseDeleted: true } + } + const row = rows[0] + return { + connectorDeleted: row.connectorArchivedAt !== null || row.connectorDeletedAt !== null, + knowledgeBaseDeleted: row.kbDeletedAt !== null, + } +} + +/** + * Decides whether deletion reconciliation may run for a sync. + * + * Reconciliation hard-deletes every stored document absent from the listing, + * so it must only run against a complete source set: + * - never on incremental syncs (they list only changed documents) + * - never when the engine truncated pagination (`listingTruncated`) — a forced + * fullSync cannot fix truncation, so it cannot override it + * - never when a provider declares its pagination non-authoritative + * - not when a connector capped its listing (`listingCapped`), unless a forced + * fullSync deliberately overrides the cap to reconcile the capped scope + */ +export function shouldReconcileDeletions( + isIncremental: boolean | undefined, + syncContext: Record | undefined, + fullSync: boolean | undefined +): boolean { + if (isIncremental) return false + if (syncContext?.listingTruncated) return false + if (syncContext?.reconciliationUnsafe) return false + return !syncContext?.listingCapped || Boolean(fullSync) +} + +/** + * Minimum number of documents a connector must still own before an empty + * listing is treated as suspect. Below it, an empty listing is far more likely + * to be a genuinely emptied source than a broken one, the blast radius of + * reconciling is a handful of documents, and any ratio-based judgement is + * statistically meaningless. + */ +const SUSPECT_LISTING_MIN_OWNED_DOCS = 3 +/** + * Minimum owned-document count before the proportional (collapse) guard + * applies. A source can legitimately shrink hard when it is small — going from + * 8 documents to 1 is ordinary editing — so the collapse guard only engages on + * corpora large enough that a near-total disappearance in a single sync is + * implausible without an upstream fault. + */ +const SUSPECT_COLLAPSE_MIN_OWNED_DOCS = 50 +/** + * A listing covering less than this fraction of the documents the connector + * still owns is treated as suspect. Deliberately far below any plausible + * bulk edit (10% means 10,000 documents collapsing to under 1,000) so normal + * housekeeping never trips it, while the partial-outage shapes seen in the + * wild — an auth wall or an interstitial served for most of a source — do. + */ +const SUSPECT_COLLAPSE_MAX_RATIO = 0.1 + +/** + * How many listed documents count toward the suspect-listing ratio. + * + * `seenExternalIds` is populated before the classification loop short-circuits + * user-excluded documents, so it counts them; the owned-document denominator + * does not, because excluded rows are filtered out of the live read. Comparing + * the two directly inflates the ratio and silently weakens the collapse guard — + * with 1,000 owned / 200 excluded, a source returning 90 documents stopped + * tripping `collapsed` entirely. Subtracting the excluded documents that were + * listed puts both sides back on the same population. + */ +export function countNonExcludedListed( + seenExternalIds: ReadonlySet, + excludedExternalIds: ReadonlySet +): number { + let excludedAndListed = 0 + for (const externalId of seenExternalIds) { + if (excludedExternalIds.has(externalId)) excludedAndListed++ + } + return seenExternalIds.size - excludedAndListed +} + +/** Why a listing is considered untrustworthy evidence of deletion. */ +export type SuspectListingReason = 'empty' | 'collapsed' + +/** + * A prior sync's listing, reconstructed from its sync-log counters. + * + * `trustworthy` is false when that run could have been an incremental listing: + * an incremental run that observed no changes is indistinguishable from a full + * run that observed nothing, and treating the former as corroboration would let + * a single bad listing confirm itself. + */ +export interface PreviousListingObservation { + listedCount: number + ownedCount: number + trustworthy: boolean +} + +/** + * Classifies a listing as untrustworthy evidence that documents were deleted. + * + * A connector that returns nothing (or almost nothing) while the knowledge base + * still holds a real corpus for it is far more likely to be broken than to be + * reporting a genuinely emptied source: observed causes include an HTTP 200 + * interstitial served instead of an index, and a source moved behind auth. + * Neither surfaces as an error, so the sync looks clean and the listing looks + * authoritative. + */ +export function classifySuspectListing( + listedCount: number, + ownedCount: number +): SuspectListingReason | null { + if (ownedCount < SUSPECT_LISTING_MIN_OWNED_DOCS) return null + if (listedCount === 0) return 'empty' + if ( + ownedCount >= SUSPECT_COLLAPSE_MIN_OWNED_DOCS && + listedCount < ownedCount * SUSPECT_COLLAPSE_MAX_RATIO + ) { + return 'collapsed' + } + return null +} + +/** + * Decides whether a suspect listing may still reconcile deletions. + * + * A suspect listing is only acted on after a consecutive sync observes the same + * thing, so a single transient upstream fault can never remove + * documents — not even reversibly, since a soft delete hides them from search + * immediately. A genuinely emptied source keeps reconciling: its second sync + * corroborates the first and tombstones everything, and a later sync — once the + * tombstoned set is again absent — completes the two-strike purge, subject to + * {@link capReconciliationDeletions}, which withholds any generation whose + * deletion count exceeds the per-sync blast-radius cap. + * + * A forced `fullSync` overrides the guard, matching its existing meaning + * elsewhere here — an explicit human request to reconcile against this listing + * right now. + */ +export function evaluateListingSafety( + listedCount: number, + ownedCount: number, + previous: PreviousListingObservation | null, + fullSync: boolean | undefined +): { reason: SuspectListingReason | null; blocked: boolean; corroborated: boolean } { + const reason = classifySuspectListing(listedCount, ownedCount) + if (!reason) return { reason: null, blocked: false, corroborated: false } + if (fullSync) return { reason, blocked: false, corroborated: false } + + const corroborated = Boolean( + previous?.trustworthy && classifySuspectListing(previous.listedCount, previous.ownedCount) + ) + return { reason, blocked: !corroborated, corroborated } +} + +/** + * Documents a reconciliation pass could actually remove. + * + * Both reads are filtered, not just the tombstoned one: the live read already + * excludes `userExcluded` rows in SQL, so filtering it again is a no-op today, + * but it keeps this count self-consistent with + * {@link partitionSyncReconciliation}, which gates deletion on the same flag for + * both lists. The result is the denominator for the deletion cap and for + * {@link classifySuspectListing}, whose numerator + * ({@link countNonExcludedListed}) ranges over the same population. + */ +export function countDeletionEligibleOwned( + existingDocs: ReconciliationDoc[], + tombstonedDocs: ReconciliationDoc[] +): number { + return ( + existingDocs.filter((d) => !d.userExcluded).length + + tombstonedDocs.filter((d) => !d.userExcluded).length + ) +} + +/** + * Operator-facing explanation of a held reconciliation pass. + * + * Stored on `knowledgeConnector.lastSyncError` because a hold is otherwise + * invisible: the sync completes normally and an operator sees an ordinary green + * run while source-removed documents stay indexed. Names the forced full sync, + * which is the documented way to apply the removals once the source is verified. + */ +export function buildReconciliationHoldNotice( + withheld: number, + cap: number, + ownedDocCount: number, + softHeld: boolean, + hardHeld: boolean +): string { + /** + * Stated per held generation. A hard-only hold withholds documents that a + * previous sync already tombstoned, so they have been invisible since then — + * telling the operator they are "still indexed" would be false. + */ + const consequence = + softHeld && hardHeld + ? 'Documents removed at the source are still indexed, and documents already pending removal were not purged.' + : softHeld + ? 'Documents removed at the source are still indexed.' + : 'Documents already pending removal were not purged; they stay hidden from search either way.' + + return ( + `Withheld ${withheld} document removal(s) — more than the ${cap} allowed per generation ` + + `in one sync of ${ownedDocCount} documents. ${consequence} ` + + 'Check the source is returning its full contents, then run a full sync to apply the removals.' + ) +} + +/** + * The document count to attribute to the previous sync when reconstructing its + * listing. + * + * `lastSyncDocCount` counts only *visible* documents, so after a pass that + * tombstoned a corpus it collapses toward 0 — and an owned count of 0 can never + * be classified as suspect, so corroboration silently became impossible and the + * two-strike purge jammed shut. Taking the larger of the recorded count and what + * the connector owns right now (tombstones included) restores the intent: the + * previous run is judged against a corpus at least as large as the one still + * present. + */ +export function resolvePreviousOwnedCount( + lastSyncDocCount: number | null | undefined, + ownedDocCount: number +): number { + return Math.max(lastSyncDocCount ?? 0, ownedDocCount) +} + +/** + * Fraction of a connector's owned documents that a single reconciliation pass + * may remove before the pass is held. + * + * {@link SUSPECT_COLLAPSE_MAX_RATIO} only questions a listing that returns under + * 10% of the corpus, which leaves every partial-outage shape between 10% and + * 100% completely unguarded: a source that serves half its documents produces a + * listing that looks perfectly healthy to every shape guard, tombstones the + * missing half, and hard-deletes it on the next pass. 25% sits well above + * ordinary housekeeping (a quarter of a corpus removed between two syncs is + * already extraordinary) and well below the outage shapes seen in the wild. + */ +const RECONCILIATION_DELETE_MAX_RATIO = 0.25 + +/** + * Deletions always permitted regardless of ratio. + * + * The ratio is meaningless on a small corpus for the same reason + * {@link SUSPECT_COLLAPSE_MIN_OWNED_DOCS} exists — removing 20 of 40 documents + * is ordinary editing — and a floor below the collapse guard's own 50-document + * threshold keeps the cap from being the binding constraint on corpora that + * guard was written to ignore. + */ +const RECONCILIATION_DELETE_MIN_ABSOLUTE = 25 + +/** Per-connector tuning for the reconciliation blast-radius cap. */ +export interface ReconciliationDeleteCapOverride { + maxRatio?: number + minAbsolute?: number +} + +/** + * Maximum number of documents one reconciliation pass may remove. + */ +export function resolveReconciliationDeleteCap( + ownedDocCount: number, + override?: ReconciliationDeleteCapOverride +): number { + const maxRatio = override?.maxRatio ?? RECONCILIATION_DELETE_MAX_RATIO + const minAbsolute = override?.minAbsolute ?? RECONCILIATION_DELETE_MIN_ABSOLUTE + return Math.max(minAbsolute, Math.floor(Math.max(ownedDocCount, 0) * maxRatio)) +} + +/** + * Caps the blast radius of one reconciliation pass. + * + * The shape guards above all reason about listings that look *broken*. Two + * confirmed data-loss paths produce listings that look perfectly healthy and so + * pass every one of them: a partial outage returning half a corpus (above the + * 10% collapse threshold), and a change to a connector's externalId derivation, + * which yields a complete, correct listing of entirely new keys — under which + * every stored document is "absent" and every listed one is new. + * + * The hold is deliberately all-or-nothing rather than a truncation to the cap: + * deleting up to the cap still destroys data, and leaves the knowledge base in a + * state no operator asked for and no later sync can reason about. For the outage + * shapes above the corpus is left intact and reconciliation resumes as soon as + * the source returns its full listing. It does NOT self-heal from a hold caused + * by genuine bulk removal: those deletions stay withheld until a `fullSync` + * applies them, which is the point — a human confirms them. + * + * The two generations are capped SEPARATELY. Soft deletes are this sync's newly + * absent documents; hard deletes are the previous generation's soft deletes, + * confirmed absent a second time and therefore already gated by this cap once. + * Summing them double-counts the older generation and, on a connector with + * steady churn, ratchets: each sync's new soft deletes plus the prior sync's + * pending hard deletes exceed the cap, the all-or-nothing hold blocks the hard + * deletes that would drain the backlog, and the backlog grows monotonically so + * the connector never reconciles again. Capping each generation against the same + * ceiling keeps the per-sync blast radius bounded without that deadlock. + * + * Note the ceiling this yields: each generation may spend the cap independently, + * so a single sync can remove up to 2x the cap — with the default ratio, about + * half the corpus, not a quarter. That is deliberate. The two generations are + * different populations: the hard deletes were already gated by this cap on the + * sync that tombstoned them, and have been invisible ever since, so confirming + * them costs no additional visible documents. The quarter-of-a-corpus figure + * describes what one sync may newly hide, which is the number that matters for a + * source that has started lying about its contents. + * + * `fullSync` bypasses the cap, matching its meaning everywhere else here — an + * explicit human request to reconcile against this listing right now, which is + * the documented escape hatch for a genuine mass deletion. + */ +export function capReconciliationDeletions( + softDeleteIds: string[], + hardDeleteIds: string[], + ownedDocCount: number, + fullSync: boolean | undefined, + override?: ReconciliationDeleteCapOverride +): { + softDeleteIds: string[] + hardDeleteIds: string[] + held: boolean + softHeld: boolean + hardHeld: boolean + withheld: number + cap: number +} { + const cap = resolveReconciliationDeleteCap(ownedDocCount, override) + const softHeld = !fullSync && softDeleteIds.length > cap + const hardHeld = !fullSync && hardDeleteIds.length > cap + + return { + softDeleteIds: softHeld ? [] : softDeleteIds, + hardDeleteIds: hardHeld ? [] : hardDeleteIds, + held: softHeld || hardHeld, + softHeld, + hardHeld, + withheld: (softHeld ? softDeleteIds.length : 0) + (hardHeld ? hardDeleteIds.length : 0), + cap, + } +} + +/** + * Reconstructs the previous completed sync's listing from its log counters. + * + * Every document the previous run listed landed in exactly one of + * added/updated/unchanged/skipped/failed, and `lastSyncDocCount` records + * how many documents the connector owned when that run finished. Documents the + * user excluded also land in `docsUnchanged`, which can only inflate the + * reconstructed listing — erring toward "the previous listing looked healthy", + * i.e. toward blocking deletions. + */ +async function loadPreviousListingObservation( + connectorId: string, + currentSyncLogId: string, + previousOwnedCount: number, + trustworthy: boolean +): Promise { + const rows = await db + .select({ + docsAdded: knowledgeConnectorSyncLog.docsAdded, + docsUpdated: knowledgeConnectorSyncLog.docsUpdated, + docsUnchanged: knowledgeConnectorSyncLog.docsUnchanged, + docsSkipped: knowledgeConnectorSyncLog.docsSkipped, + docsFailed: knowledgeConnectorSyncLog.docsFailed, + }) + .from(knowledgeConnectorSyncLog) + .where( + and( + eq(knowledgeConnectorSyncLog.connectorId, connectorId), + eq(knowledgeConnectorSyncLog.status, 'completed'), + ne(knowledgeConnectorSyncLog.id, currentSyncLogId) + ) + ) + .orderBy(desc(knowledgeConnectorSyncLog.startedAt)) + .limit(1) + + const previous = rows[0] + if (!previous) return null + + return { + listedCount: + previous.docsAdded + + previous.docsUpdated + + previous.docsUnchanged + + previous.docsSkipped + + previous.docsFailed, + ownedCount: previousOwnedCount, + trustworthy, + } +} + +/** + * Decides whether a sync should use the connector's incremental listing. + * + * A pending-removal document only surfaces in an incremental listing if its + * content changed since last sync — an unchanged-but-still-present document + * never appears in an incremental delta at all, so it could never be + * resurrected and would stay tombstoned indefinitely on a connector that runs + * incrementally from here on. `hasTombstonedDocs` forces a full listing + * whenever any pending-removal document exists for this connector, so every + * one of them gets a real resurrect-or-confirm decision on this sync. + */ +export function shouldRunIncrementalSync( + supportsIncrementalSync: boolean | undefined, + syncMode: string | null | undefined, + fullSync: boolean | undefined, + rehydrate: boolean | undefined, + hasTombstonedDocs: boolean, + lastSyncAt: string | Date | null | undefined +): boolean { + return Boolean( + supportsIncrementalSync && + syncMode !== 'full' && + !fullSync && + !hasTombstonedDocs && + !rehydrate && + lastSyncAt != null + ) +} + +/** + * A stored document's identity, as read back for reconciliation. + * + * `userExcluded` is required, not optional. Both reads project it, so the + * deletion guards in {@link partitionSyncReconciliation} enforce something on + * their own rather than restating a filter the SQL already applied — if that + * filter were ever dropped, the guard would still hold. An optional flag made + * the guard a silent no-op on any read that forgot to select it. + */ +type ReconciliationDoc = { id: string; externalId: string | null; userExcluded: boolean } + +/** + * Partitions a connector's stored documents against the current listing into + * the three reconciliation actions. + * + * A document absent from a normal (non-fullSync) listing is never purged + * immediately — an empty or shrunken listing can equally mean a transient + * source outage, and a single bad observation must never cause an + * irreversible mass deletion. It is instead marked pending-removal + * (`softDeleteIds`), and only becomes eligible for hard deletion + * (`hardDeleteIds`) once a *later* sync confirms it's still absent — i.e. it + * was already pending-removal (`tombstonedDocs`) coming into this sync. A + * document that reappears while pending-removal is resurrected + * (`resurrectIds`) regardless of `fullSync`, since presence — unlike absence — + * is trustworthy evidence even from a partial listing. A document whose + * content refresh was attempted but failed (`failedExternalIds`) is excluded + * from resurrection even though it was seen — surfacing it now would show + * known-stale pre-tombstone content; it stays tombstoned for a later sync to + * retry. + * + * A forced `fullSync` is an explicit request to reconcile right now: it skips + * the grace period and purges everything absent in one pass. + * + * A `userExcluded` document is never deletion-eligible — the user asked to keep + * the row — but it stays fully resurrection-eligible. The distinction matters: + * `userExcluded` and `enabled` gate visibility on their own in every retrieval + * path, so resurrecting one never re-indexes it; it only clears `deletedAt`. + * Withholding resurrection instead would strand the row permanently, since the + * connector-document listing and the restore mutation both require + * `deletedAt IS NULL` — leaving it invisible, unrestorable, and (by this very + * guard) undeletable. + */ +export function partitionSyncReconciliation( + existingDocs: ReconciliationDoc[], + tombstonedDocs: ReconciliationDoc[], + seenExternalIds: Set, + failedExternalIds: Set, + fullSync: boolean | undefined +): { resurrectIds: string[]; softDeleteIds: string[]; hardDeleteIds: string[] } { + const resurrectIds = tombstonedDocs + .filter( + (d) => + d.externalId && seenExternalIds.has(d.externalId) && !failedExternalIds.has(d.externalId) + ) + .map((d) => d.id) + const liveMissingIds = existingDocs + .filter((d) => d.externalId && !d.userExcluded && !seenExternalIds.has(d.externalId)) + .map((d) => d.id) + const tombstonedStillMissingIds = tombstonedDocs + .filter((d) => d.externalId && !d.userExcluded && !seenExternalIds.has(d.externalId)) + .map((d) => d.id) + + if (fullSync) { + return { + resurrectIds, + softDeleteIds: [], + hardDeleteIds: [...liveMissingIds, ...tombstonedStillMissingIds], + } + } + return { resurrectIds, softDeleteIds: liveMissingIds, hardDeleteIds: tombstonedStillMissingIds } +} + +/** + * Re-filters the three reconciliation ID lists against a fresh ownership + * snapshot taken under the connector's `FOR UPDATE` lock, dropping any + * document a concurrent "delete connector, keep documents" request already + * detached (its `connectorId` no longer matches) since the lists were first + * computed. + */ +export function filterStillOwnedReconciliationIds( + resurrectIds: string[], + softDeleteIds: string[], + hardDeleteIds: string[], + stillOwnedIds: Set +): { resurrectIds: string[]; softDeleteIds: string[]; hardDeleteIds: string[] } { + return { + resurrectIds: resurrectIds.filter((id) => stillOwnedIds.has(id)), + softDeleteIds: softDeleteIds.filter((id) => stillOwnedIds.has(id)), + hardDeleteIds: hardDeleteIds.filter((id) => stillOwnedIds.has(id)), + } +} + +/** What a change-feed pass needs from the engine that runs it. */ +export interface ChangeFeedPassInput { + connectorId: string + connectorConfig: { listChanges: NonNullable } + sourceConfig: Record + syncContext: Record + /** Where the feed was last left. */ + cursor: string + beforePage: () => Promise + getAccessToken: (pageNum: number) => Promise + deadlineAt?: number + maxPages?: number +} + +export interface ChangeFeedPassResult { + /** The latest stub of every item the feed reported as present, in feed order. */ + upserts: ExternalDocument[] + /** Items whose last word from the feed was a removal. */ + removedExternalIds: string[] + /** Where the next read resumes: past every page this pass consumed. */ + cursor: string + /** False when pagination stopped before the feed was drained. */ + exhausted: boolean + budgetAborted: boolean +} + +/** + * Reads a change feed to exhaustion, the page cap, or the deadline. Each item + * keeps only its last change, so something removed and re-shared inside one + * pass reads as present. The returned cursor sits past every page that was + * read, so an interrupted pass never replays what it already applied. + */ +export async function runChangeFeedPass(input: ChangeFeedPassInput): Promise { + const { connectorId, connectorConfig, sourceConfig, syncContext } = input + const maxPages = input.maxPages ?? MAX_PAGES + const latest = new Map() + let retainedSourcePayloadBytes = 0 + let cursor = input.cursor + let hasMore = true + let budgetAborted = false + + for (let pageNum = 0; hasMore && pageNum < maxPages; pageNum++) { + await input.beforePage() + + if (input.deadlineAt !== undefined && Date.now() >= input.deadlineAt) { + budgetAborted = true + break + } + + const accessToken = await input.getAccessToken(pageNum) + const page = await connectorConfig.listChanges(accessToken, sourceConfig, cursor, syncContext) + + const upserts: ExternalDocument[] = [] + for (const change of page.changes) { + if (change.kind === 'upsert') upserts.push(change.document) + } + if (!sourcePageFitsSyncWorkingSet(latest.size, upserts.length)) { + throw new ConnectorSyncWorkingSetLimitError(connectorId, 'change feed') + } + retainedSourcePayloadBytes = addSourcePagePayloadBytes(retainedSourcePayloadBytes, upserts) + for (const change of page.changes) latest.set(change.externalId, change) + + cursor = page.nextCursor + hasMore = page.hasMore + } + + const result: ChangeFeedPassResult = { + upserts: [], + removedExternalIds: [], + cursor, + exhausted: !hasMore, + budgetAborted, + } + for (const change of latest.values()) { + if (change.kind === 'upsert') result.upserts.push(change.document) + else result.removedExternalIds.push(change.externalId) + } + return result +} + +/** What a listing pass needs from the engine that runs it. */ +export interface ListingPassInput { + connectorId: string + connectorConfig: Pick + sourceConfig: Record + /** Per-run mutable state the connector caches token-derived lookups in. */ + syncContext: Record + /** Incremental watermark handed to the connector; undefined lists everything. */ + lastSyncAt: Date | undefined + /** Runs before every page is fetched; the content engine heartbeats here. */ + beforePage: () => Promise + /** + * The token to list page `pageNum` with. The content engine re-resolves an + * OAuth token for every page after the first so a long listing outlives a + * short-lived access token. + */ + getAccessToken: (pageNum: number) => Promise + /** Wall-clock instant (ms since epoch) after which no further page is fetched. */ + deadlineAt?: number + maxPages?: number +} + +export interface ListingPassResult { + documents: ExternalDocument[] + /** False when pagination stopped before the source was exhausted. */ + exhausted: boolean + /** True when the deadline stopped pagination; implies `exhausted` is false. */ + budgetAborted: boolean +} + +/** + * Pages a connector's listing to exhaustion, the page cap, the deadline, or a + * missing cursor, enforcing the working-set and retained-payload limits as it + * goes. The caller decides what an unexhausted listing means: the content + * engine marks it capped and truncated so deletion reconciliation is skipped. + */ +export async function runListingPass(input: ListingPassInput): Promise { + const { connectorId, connectorConfig, sourceConfig, syncContext, lastSyncAt } = input + const maxPages = input.maxPages ?? MAX_PAGES + const externalDocs: ExternalDocument[] = [] + let retainedSourcePayloadBytes = 0 + let cursor: string | undefined + let hasMore = true + let budgetAborted = false + + for (let pageNum = 0; hasMore && pageNum < maxPages; pageNum++) { + /** + * Listing is where a large source spends most of its wall clock — the + * batch loop does not start until every page has been fetched — so + * without this a big listing outran the TTL and was reclaimed as a hard + * failure, which is the exact ratchet the heartbeat exists to prevent. + */ + await input.beforePage() + + if (input.deadlineAt !== undefined && Date.now() >= input.deadlineAt) { + budgetAborted = true + break + } + + const accessToken = await input.getAccessToken(pageNum) + + const page = await connectorConfig.listDocuments( + accessToken, + sourceConfig, + cursor, + syncContext, + lastSyncAt + ) + if (page.reconciliationSafe === false) { + syncContext.reconciliationUnsafe = true + } + if (!sourcePageFitsSyncWorkingSet(externalDocs.length, page.documents.length)) { + throw new ConnectorSyncWorkingSetLimitError(connectorId, 'source listing') + } + retainedSourcePayloadBytes = addSourcePagePayloadBytes( + retainedSourcePayloadBytes, + page.documents + ) + externalDocs.push(...page.documents) + + if (page.hasMore && !page.nextCursor) { + logger.warn('Source returned hasMore=true with no cursor, stopping pagination', { + connectorId, + pageNum, + docsSoFar: externalDocs.length, + }) + break + } + + cursor = page.nextCursor + hasMore = page.hasMore + } + + return { documents: externalDocs, exhausted: !hasMore, budgetAborted } +} + +/** A live, non-excluded document the connector owns, as read for classification. */ +export interface OwnedLiveDocument { + id: string + externalId: string | null + contentHash: string | null + storageKey: string | null + userExcluded: boolean +} + +/** A pending-removal document the connector owns. */ +export interface OwnedTombstonedDocument extends OwnedLiveDocument { + deletedAt: Date | null +} + +/** Everything the connector owns, loaded once per run under one memory budget. */ +export interface OwnedCorpus { + existingDocs: OwnedLiveDocument[] + tombstonedDocs: OwnedTombstonedDocument[] + /** Listed external ids whose hydration is short-circuited: the user chose "keep but don't index". */ + excludedExternalIds: Set + priorByExternalId: Map +} + +/** + * Loads the connector's owned corpus: live documents, tombstones, and the + * user-excluded external ids. + * + * Loaded sequentially with a shared sentinel budget. Three concurrent + * `SELECT`s each capped independently could still materialize three times + * the intended working set before the overflow was detected. + */ +export async function loadOwnedCorpus(connectorId: string): Promise { + const existingDocs = await db + .select({ + id: document.id, + externalId: document.externalId, + contentHash: document.contentHash, + storageKey: document.storageKey, + /** + * Projected as well as filtered: the SQL predicate and the in-memory guard in + * partitionSyncReconciliation must both hold, so dropping either one alone cannot make + * an excluded document deletable. + */ + userExcluded: document.userExcluded, + }) + .from(document) + .where( + and( + eq(document.connectorId, connectorId), + /** + * A user's explicit "keep but don't index" choice must never make a document eligible + * for reconciliation deletion: it is deliberately never refreshed, so its absence from + * a listing says nothing. + */ + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNull(document.deletedAt) + ) + ) + .limit(syncWorkingSetQueryLimit(0)) + assertSyncWorkingSetWithinLimit(connectorId, 0, existingDocs.length) + + /** + * Documents already marked pending-removal by a prior sync's reconciliation: absent from the + * source once, not yet absent twice in a row. Including them in classification lets a document + * that reappears be recognized as existing (resurrected) rather than re-added. + */ + const tombstonedDocs = await db + .select({ + id: document.id, + externalId: document.externalId, + contentHash: document.contentHash, + storageKey: document.storageKey, + deletedAt: document.deletedAt, + /** + * Gates hard deletion in partitionSyncReconciliation without gating resurrection. + */ + userExcluded: document.userExcluded, + }) + .from(document) + .where( + and( + eq(document.connectorId, connectorId), + /** + * Load both included and user-excluded tombstones. Excluded tombstones are never + * deletion-eligible, but they must remain resurrection-eligible when their source + * document reappears or the row becomes permanently invisible and unrestorable. + */ + isNull(document.archivedAt), + isNotNull(document.deletedAt) + ) + ) + .limit(syncWorkingSetQueryLimit(existingDocs.length)) + assertSyncWorkingSetWithinLimit(connectorId, existingDocs.length, tombstonedDocs.length) + + /** + * Live user-excluded rows form the third disjoint population in the shared memory budget. + * User-excluded tombstones were loaded above so source presence can clear their deletion marker; + * they are added to `excludedExternalIds` below to keep hydration short-circuited. + */ + const loadedOwnedDocs = existingDocs.length + tombstonedDocs.length + const excludedDocs = await db + .select({ externalId: document.externalId }) + .from(document) + .where( + and( + eq(document.connectorId, connectorId), + eq(document.userExcluded, true), + isNull(document.archivedAt), + isNull(document.deletedAt) + ) + ) + .limit(syncWorkingSetQueryLimit(loadedOwnedDocs)) + assertSyncWorkingSetWithinLimit(connectorId, loadedOwnedDocs, excludedDocs.length) + + const excludedExternalIds = new Set( + [ + ...excludedDocs.map((doc) => doc.externalId), + ...tombstonedDocs.filter((doc) => doc.userExcluded).map((doc) => doc.externalId), + ].filter((externalId): externalId is string => Boolean(externalId)) + ) + + const priorByExternalId = new Map( + [...existingDocs, ...tombstonedDocs] + .filter((d) => d.externalId !== null) + .map((d) => [d.externalId!, d]) + ) + + return { existingDocs, tombstonedDocs, excludedExternalIds, priorByExternalId } +} + +/** The per-run bookkeeping the classification and persistence stages share. */ +export interface SyncRunState { + result: SyncResult + /** Every external id the listing produced, deduplicated at first sight. */ + seenExternalIds: Set + /** + * externalIds whose content was never verified as current: a hydration + * error, a rejected write, a fulfilled-but-unusable hydration (skipped as + * oversized, or an empty re-fetch), a listing-time skippedReason + * short-circuit, or empty non-deferred content (`drop`) — all fall back to + * either keeping the stored content as last-known-good or discarding the + * listing entry outright, without ever comparing or refreshing content. + * That's fine for an already-visible document, but for a tombstoned one it + * means we still don't have confirmed-current content — so this excludes + * them from resurrection: a tombstoned document whose refresh didn't + * actually land must stay tombstoned rather than come back visible while + * still serving stale pre-tombstone content. + */ + failedExternalIds: Set +} + +/** Fresh bookkeeping for one run. */ +export function createSyncRunState(result: SyncResult): SyncRunState { + return { result, seenExternalIds: new Set(), failedExternalIds: new Set() } +} + +/** + * Turns the listing into the operations that need content work, counting the + * documents that need none. Duplicated external ids are seen once; excluded + * documents count as unchanged without ever being compared. + */ +export function classifyListing(input: { + externalDocs: ExternalDocument[] + corpus: Pick + forceRehydrate: boolean + state: SyncRunState +}): DocOp[] { + const { externalDocs, corpus, forceRehydrate } = input + const { result, seenExternalIds, failedExternalIds } = input.state + + const pendingOps: DocOp[] = [] + for (const extDoc of externalDocs) { + if (seenExternalIds.has(extDoc.externalId)) continue + seenExternalIds.add(extDoc.externalId) + + if (corpus.excludedExternalIds.has(extDoc.externalId)) { + result.docsUnchanged++ + continue + } + + const existing = corpus.priorByExternalId.get(extDoc.externalId) + const classification = classifyExternalDoc(extDoc, existing, forceRehydrate) + + switch (classification.type) { + case 'skip': + pendingOps.push({ + type: 'skip', + existingId: classification.existingId, + extDoc, + }) + break + case 'drop': + // Empty, non-deferred content is never usable. If this was a + // reappearing tombstoned document, its content was never verified as + // current — see failedExternalIds. + if (existing) { + recordUnverifiedExistingRefresh(result, failedExternalIds, extDoc.externalId) + } + logger.info(`Skipping empty document: ${extDoc.title}`, { + externalId: extDoc.externalId, + }) + break + case 'add': + pendingOps.push({ type: 'add', extDoc }) + break + case 'update': + pendingOps.push({ type: 'update', existingId: classification.existingId, extDoc }) + break + case 'unchanged': + // A listing-time skippedReason short-circuits classification before + // the hash comparison, so this is "kept as last-known-good", not a + // verified-unchanged match — same as the deferred-hydration + // equivalent. A genuine hash match never sets skippedReason, + // so this only fires for the short-circuited case. + if (extDoc.skippedReason && existing) { + recordUnverifiedExistingRefresh(result, failedExternalIds, extDoc.externalId) + } else { + result.docsUnchanged++ + } + break + } + } + return pendingOps +} + +/** How deferred content is fetched; each engine supplies the identity it fetches with. */ +export interface DocOpHydration { + /** Runs once per batch that has deferred documents, before any of them is fetched. */ + beforeHydration?: () => Promise + getDocument: (externalId: string) => Promise +} + +/** What the persistence stage needs from the engine that runs it. */ +export interface ProcessDocOpsInput { + connectorId: string + connector: { knowledgeBaseId: string; connectorType: string } + sourceConfig: Record + kbOwner: KnowledgeBaseOwner + billingAttribution: BillingAttributionSnapshot + pendingOps: DocOp[] + corpus: Pick + forceRehydrate: boolean + state: SyncRunState + hydration: DocOpHydration + lease: Pick + /** Who may read the documents this pass writes. */ + documentAccess: SyncDocumentAccess +} + +/** + * Hydrates, stores, and dispatches the pending operations in batches bounded + * by both count and in-flight content bytes. Every failure is counted on the + * run state rather than thrown, except a provider rate limit, which ends the + * run so the connector backs off, and a lost lease, which ends it so no + * further write lands beside the replacement run's. + */ +export async function processDocOps(input: ProcessDocOpsInput): Promise { + const { + connectorId, + connector, + sourceConfig, + kbOwner, + billingAttribution, + forceRehydrate, + documentAccess, + } = input + const { priorByExternalId } = input.corpus + const { result, failedExternalIds } = input.state + + // Batch by both count and summed content bytes so a few large files near the + // per-file cap never hydrate/upload together and exhaust the worker heap. + const batches = chunkOpsByByteBudget( + input.pendingOps, + CONTENT_INFLIGHT_BUDGET_BYTES, + SYNC_BATCH_SIZE + ) + for (const rawBatch of batches) { + const presence = await checkSyncTargetPresence(connectorId, connector.knowledgeBaseId) + if (presence.connectorDeleted) { + throw new ConnectorDeletedException(connectorId) + } + if (presence.knowledgeBaseDeleted) { + throw new Error(`Knowledge base ${connector.knowledgeBaseId} was deleted during sync`) + } + + // After liveness: a deleted connector must raise ConnectorDeletedException + // and run its cleanup, not be reported as a lost lock. + await input.lease.beatIfDue() + + // Oversized/skipped docs become visible `failed` rows (never silent). They are + // flagged either at listing time (skip ops here) or discovered only at fetch + // time during hydration below; both are collected and persisted after hydration. + const skipOps = rawBatch.filter((op) => op.type === 'skip') + const skippedRetryHashUpdates: Array<{ + existingId: string + externalId: string + contentHash: string + }> = [] + + const contentOps = rawBatch.filter((op) => op.type !== 'skip') + const deferredOps = contentOps.filter((op) => op.extDoc.contentDeferred) + const readyOps = contentOps.filter((op) => !op.extDoc.contentDeferred) + + if (deferredOps.length > 0) { + await input.hydration.beforeHydration?.() + + const hydrated = await Promise.allSettled( + deferredOps.map(async (op) => { + const fullDoc = requireHydratedListedDocument( + await input.hydration.getDocument(op.extDoc.externalId), + op.extDoc.externalId + ) + // A connector may only learn a file is too large at fetch time (its + // listing has no size). Surface that as a failed row for new files; keep + // already-indexed files as last-known-good rather than downgrading them. + if (fullDoc?.skippedReason) { + if (op.type === 'add') { + skipOps.push({ + type: 'skip', + extDoc: mergeHydratedSkippedDocument(op.extDoc, fullDoc), + }) + } else if (op.type === 'update') { + const existing = priorByExternalId.get(op.extDoc.externalId) + if (existing && shouldReplaceExistingWithSkippedDocument(existing, fullDoc)) { + skipOps.push({ + type: 'skip', + existingId: op.existingId, + extDoc: mergeHydratedSkippedDocument(op.extDoc, fullDoc), + }) + } else { + if (fullDoc.skippedRetryContentHash) { + skippedRetryHashUpdates.push({ + existingId: op.existingId, + externalId: op.extDoc.externalId, + contentHash: fullDoc.skippedRetryContentHash, + }) + } + /** Preserve last-known-good content and replay the unverified source change. */ + recordUnverifiedExistingRefresh(result, failedExternalIds, op.extDoc.externalId) + } + } + return null + } + if (!hasIndexablePayload(fullDoc)) { + /** An empty refresh cannot replace or advance past last-known-good content. */ + if (op.type === 'update') { + recordUnverifiedExistingRefresh(result, failedExternalIds, op.extDoc.externalId) + } + return null + } + const hydratedHash = fullDoc.contentHash ?? op.extDoc.contentHash + /** + * Normally an update whose hydrated hash matches the stored hash is a + * no-op (content unchanged). On a forced re-hydration the hash is + * version-based and cannot reflect the rendered-dependency change we are + * refreshing for, so re-index unconditionally instead of skipping. + */ + if ( + op.type === 'update' && + !forceRehydrate && + priorByExternalId.get(op.extDoc.externalId)?.contentHash === hydratedHash + ) { + result.docsUnchanged++ + return null + } + return { ...op, extDoc: mergeHydratedDocument(op.extDoc, fullDoc, hydratedHash) } + }) + ) + + const rateLimitFailure = hydrated.find( + (outcome): outcome is PromiseRejectedResult => + outcome.status === 'rejected' && isRateLimitError(outcome.reason) + ) + if (rateLimitFailure) { + throw rateLimitFailure.reason + } + + for (let i = 0; i < hydrated.length; i++) { + const outcome = hydrated[i] + if (outcome.status === 'fulfilled' && outcome.value) { + readyOps.push(outcome.value) + } else if (outcome.status === 'rejected') { + result.docsFailed++ + failedExternalIds.add(deferredOps[i].extDoc.externalId) + logger.error('Failed to hydrate deferred document', { + connectorId, + externalId: deferredOps[i].extDoc.externalId, + error: getErrorMessage(outcome.reason), + }) + } + } + } + + /** + * Hydration above may have outlasted the lease. Nothing from this batch is + * written until the run proves it still owns the connector, and every + * write below proves it again inside its own transaction, so a run that + * was replaced meanwhile cannot land stale content or queue processing + * over the replacement's. + */ + await input.lease.beatLive() + + if (skippedRetryHashUpdates.length > 0) { + try { + const missedExternalIds = await persistSkippedRetryHashes( + connector.knowledgeBaseId, + connectorId, + skippedRetryHashUpdates, + input.lease + ) + if (missedExternalIds.length > 0) { + logger.warn('Skipped retry hashes were not persisted for detached documents', { + connectorId, + externalIds: missedExternalIds, + }) + } + } catch (error) { + logger.error('Failed to persist skipped document retry hashes', { + connectorId, + count: skippedRetryHashUpdates.length, + error: toError(error).message, + }) + throw error + } + } + + if (skipOps.length > 0) { + try { + const recorded = await persistSkippedDocuments( + connector.knowledgeBaseId, + connectorId, + connector.connectorType, + skipOps, + sourceConfig, + documentAccess, + input.lease + ) + result.docsSkipped += recorded + } catch (error) { + if (error instanceof SyncLockLostException) throw error + /** + * The source items were intentionally skipped, but failing to persist their visible + * failed rows is an actual sync failure. + */ + result.docsFailed += skipOps.length + for (const op of skipOps) { + failedExternalIds.add(op.extDoc.externalId) + } + logger.error('Failed to record skipped documents', { + connectorId, + count: skipOps.length, + error: toError(error).message, + }) + } + } + + const batch = readyOps + + const settled = await Promise.allSettled( + batch.map((op) => { + if (op.type === 'add') { + return addDocument( + connector.knowledgeBaseId, + connectorId, + connector.connectorType, + op.extDoc, + kbOwner, + sourceConfig, + documentAccess, + input.lease + ) + } + return updateDocument( + op.existingId, + connector.knowledgeBaseId, + connectorId, + connector.connectorType, + op.extDoc, + kbOwner, + sourceConfig, + documentAccess, + input.lease + ) + }) + ) + + const leaseLost = settled.find( + (outcome): outcome is PromiseRejectedResult => + outcome.status === 'rejected' && outcome.reason instanceof SyncLockLostException + ) + if (leaseLost) throw leaseLost.reason + + const batchDocs: DocumentData[] = [] + for (let j = 0; j < settled.length; j++) { + const outcome = settled[j] + if (outcome.status === 'fulfilled') { + batchDocs.push(outcome.value) + if (batch[j].type === 'add') result.docsAdded++ + else result.docsUpdated++ + } else { + result.docsFailed++ + failedExternalIds.add(batch[j].extDoc.externalId) + logger.error('Failed to process document', { + connectorId, + externalId: batch[j].extDoc.externalId, + error: getErrorMessage(outcome.reason), + }) + } + } + + if (batchDocs.length > 0) { + result.processingDispatch.requested += batchDocs.length + try { + const dispatch = await processDocumentsWithQueue( + batchDocs, + connector.knowledgeBaseId, + {}, + generateId(), + billingAttribution, + { connectorId, stillHeld: input.lease.stillHeld } + ) + result.processingDispatch.accepted += dispatch.accepted + result.processingDispatch.failed += dispatch.failed + } catch (error) { + if (error instanceof SyncLockLostException) throw error + result.processingDispatch.failed += batchDocs.length + logger.warn('Failed to enqueue batch for processing — will retry on next sync', { + connectorId, + count: batchDocs.length, + error: toError(error).message, + }) + } + } + } +} + +/** What deletion reconciliation needs from the engine that runs it. */ +export interface ReconcileDeletionsInput { + connectorId: string + connector: { + knowledgeBaseId: string + connectorType: string + lastSyncDocCount: number | null + syncMode: string | null + } + connectorConfig: Pick + /** The run's sync-log id, which is also the lock token guarding every delete. */ + syncLogId: string + syncContext: Record + isIncremental: boolean + fullSync: boolean | undefined + corpus: Pick + state: SyncRunState + lease: SyncRunLease +} + +/** + * Resurrects documents that reappeared and, when the listing is trustworthy + * evidence of absence, tombstones newly absent documents and purges those + * absent twice in a row — all under the shape guards and the blast-radius cap. + * Returns the operator-facing notice of a held pass, or null when nothing was + * withheld. + */ +export async function reconcileDeletions(input: ReconcileDeletionsInput): Promise { + const { connectorId, connector, connectorConfig, syncLogId, syncContext, isIncremental } = input + const { existingDocs, tombstonedDocs, excludedExternalIds } = input.corpus + const { result, seenExternalIds, failedExternalIds } = input.state + const fullSync = input.fullSync + + const { resurrectIds, softDeleteIds, hardDeleteIds } = partitionSyncReconciliation( + existingDocs, + tombstonedDocs, + seenExternalIds, + failedExternalIds, + fullSync + ) + + let reconcileDeletionsAllowed = shouldReconcileDeletions(isIncremental, syncContext, fullSync) + + /** + * Counted over deletion-eligible rows on both sides. The live read filters + * excluded documents in SQL; the tombstoned read only projects the flag, so + * excluded tombstones must be dropped here or they inflate a denominator + * governing a population they are not part of. Matches `listedDocCount`, + * which `countNonExcludedListed` already puts on the same footing. + */ + const ownedDocCount = countDeletionEligibleOwned(existingDocs, tombstonedDocs) + /** + * Counted over the same population as `ownedDocCount`: excluded documents + * are absent from the live read, so they must not inflate the numerator. + */ + const listedDocCount = countNonExcludedListed(seenExternalIds, excludedExternalIds) + /** + * Backstop shared by every connector: a listing that reports (almost) + * nothing while this connector still owns a real corpus is treated as a + * fault, not as evidence of deletion, until a consecutive sync sees the + * same thing. Only evaluated when reconciliation would otherwise run, so + * healthy syncs pay nothing and no existing gate is loosened. + */ + if (reconcileDeletionsAllowed && classifySuspectListing(listedDocCount, ownedDocCount)) { + const previousObservation = await loadPreviousListingObservation( + connectorId, + syncLogId, + resolvePreviousOwnedCount(connector.lastSyncDocCount, ownedDocCount), + !connectorConfig.supportsIncrementalSync || connector.syncMode === 'full' + ) + const listingSafety = evaluateListingSafety( + listedDocCount, + ownedDocCount, + previousObservation, + fullSync + ) + logger.warn('Suspect connector listing detected', { + connectorId, + connectorType: connector.connectorType, + reason: listingSafety.reason, + listedDocs: listedDocCount, + listedDocsIncludingExcluded: seenExternalIds.size, + ownedDocs: ownedDocCount, + liveDocs: existingDocs.length, + tombstonedDocs: tombstonedDocs.length, + previousListedDocs: previousObservation?.listedCount ?? null, + previousObservationTrusted: previousObservation?.trustworthy ?? false, + deletionReconciliation: listingSafety.blocked ? 'skipped' : 'proceeding', + syncRunId: syncContext.syncRunId, + }) + if (listingSafety.blocked) { + reconcileDeletionsAllowed = false + } + } + + /** + * Last word after every shape guard: even a listing that looks entirely + * healthy may not remove an implausible share of the corpus in one pass. + * Applied here so it covers both the soft-delete UPDATE and the + * `hardDeleteDocuments` call below. + */ + const capped = capReconciliationDeletions( + reconcileDeletionsAllowed ? softDeleteIds : [], + reconcileDeletionsAllowed ? hardDeleteIds : [], + ownedDocCount, + fullSync + ) + /** + * Surfaced on the connector so a held pass is visible to an operator rather + * than only in logs: without it the sync completes green, clears + * `lastSyncError`, and source-removed documents stay indexed with no signal. + * Written through the success update at the end of the run rather than + * here — that update sets `lastSyncError: null` unconditionally and would + * otherwise clobber this within the same sync. `status` is deliberately left + * `active`: the sync itself succeeded, and marking the connector broken + * would stop it syncing at all. + */ + let reconciliationHoldNotice: string | null = null + if (capped.held) { + reconciliationHoldNotice = buildReconciliationHoldNotice( + capped.withheld, + capped.cap, + ownedDocCount, + capped.softHeld, + capped.hardHeld + ) + logger.error('Reconciliation deletions held — exceeds per-sync blast-radius cap', { + connectorId, + connectorType: connector.connectorType, + withheld: capped.withheld, + softHeld: capped.softHeld, + hardHeld: capped.hardHeld, + requestedSoft: softDeleteIds.length, + requestedHard: hardDeleteIds.length, + cap: capped.cap, + ownedDocCount, + listedCount: listedDocCount, + syncRunId: syncContext.syncRunId, + }) + } + + const gatedSoftDeleteIds = capped.softDeleteIds + const gatedHardDeleteIds = capped.hardDeleteIds + + const candidateIds = [...new Set([...resurrectIds, ...gatedSoftDeleteIds, ...gatedHardDeleteIds])] + + let safeResurrectIds: string[] = [] + let safeSoftDeleteIds: string[] = [] + let safeHardDeleteIds: string[] = [] + + if (candidateIds.length > 0) { + /** + * A concurrent "delete connector, keep documents" request detaches these + * same documents (connectorId set to NULL) under the same FOR UPDATE lock + * the DELETE route takes on this connector row. Taking that lock here + * serializes the two requests: whichever commits first wins, and the + * loser's re-check below sees the up-to-date connectorId and skips any + * document the other request already claimed — instead of resurrecting or + * deleting a document that another request just detached (and possibly + * already billed) as a standalone KB entry. + */ + await db.transaction(async (tx) => { + const [activeKnowledgeBase] = await tx + .select({ id: knowledgeBase.id }) + .from(knowledgeBase) + .where( + and(eq(knowledgeBase.id, connector.knowledgeBaseId), isNull(knowledgeBase.deletedAt)) + ) + .for('update') + if (!activeKnowledgeBase) throw new SyncLockLostException(connectorId) + + const [heldSyncLock] = await tx + .select({ id: knowledgeConnector.id }) + .from(knowledgeConnector) + .where(input.lease.stillHeld()) + .for('update') + if (!heldSyncLock) throw new SyncLockLostException(connectorId) + + const stillOwned = new Set( + ( + await tx + .select({ id: document.id }) + .from(document) + .where( + and( + inArray(document.id, candidateIds), + eq(document.connectorId, connectorId), + isNull(document.archivedAt) + ) + ) + ).map((d) => d.id) + ) + + const stillOwnedResult = filterStillOwnedReconciliationIds( + resurrectIds, + gatedSoftDeleteIds, + gatedHardDeleteIds, + stillOwned + ) + safeResurrectIds = stillOwnedResult.resurrectIds + safeSoftDeleteIds = stillOwnedResult.softDeleteIds + safeHardDeleteIds = stillOwnedResult.hardDeleteIds + + /** + * A document reappearing at the source is trustworthy evidence on its + * own — unlike absence, presence never depends on the listing being + * complete — so resurrection runs unconditionally, even on an + * incremental or otherwise gated sync. + */ + if (safeResurrectIds.length > 0) { + await tx + .update(document) + .set({ deletedAt: null }) + .where( + and( + inArray(document.id, safeResurrectIds), + eq(document.connectorId, connectorId), + isNull(document.archivedAt), + isNotNull(document.deletedAt) + ) + ) + } + if (safeSoftDeleteIds.length > 0) { + await tx + .update(document) + .set({ deletedAt: new Date() }) + .where( + and( + inArray(document.id, safeSoftDeleteIds), + eq(document.connectorId, connectorId), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNull(document.deletedAt) + ) + ) + } + }) + } + + if (safeResurrectIds.length > 0) { + logger.info(`Resurrected ${safeResurrectIds.length} documents that reappeared at the source`, { + connectorId, + }) + } + if (safeSoftDeleteIds.length > 0) { + logger.info( + `Marked ${safeSoftDeleteIds.length} documents pending removal — absent from source, confirming on next sync`, + { connectorId } + ) + } + for (let i = 0; i < safeHardDeleteIds.length; i += HARD_DELETE_CHUNK_SIZE) { + await input.lease.beatIfDue() + try { + result.docsDeleted += await hardDeleteDocuments( + safeHardDeleteIds.slice(i, i + HARD_DELETE_CHUNK_SIZE), + syncLogId, + connectorId, + connector.knowledgeBaseId, + { + connectorId, + knowledgeBaseId: connector.knowledgeBaseId, + syncLockToken: syncLogId, + } + ) + } catch (error) { + if (error instanceof ConnectorSyncDeletionGuardError) { + throw new SyncLockLostException(connectorId) + } + throw error + } + } + + return reconciliationHoldNotice +} + +/** What the stuck-document sweep needs from the engine that runs it. */ +export interface SweepStuckDocumentsInput { + connectorId: string + knowledgeBaseId: string + /** Documents uploaded at or after this instant belong to the current run and are left alone. */ + syncStartedAt: Date + /** Documents older than this are outside the retry window. */ + retryCutoff: Date + billingAttribution: BillingAttributionSnapshot + result: SyncResult + lease: SyncRunLease +} + +/** + * Reclaims documents this connector left unfinished: a terminated attempt, a + * dispatch that never produced a run, or a run abandoned mid-processing. + * + * The query applies each status's age rule before the candidate limit, so + * recently requeued old uploads cannot hide genuinely overdue work. The same + * rules are evaluated again after candidate rows are locked. Skipped + * documents are content-less `failed` rows with no storage key and therefore + * remain excluded outright. + */ +export async function sweepStuckDocuments(input: SweepStuckDocumentsInput): Promise { + const { connectorId, knowledgeBaseId, syncStartedAt, retryCutoff, billingAttribution, result } = + input + + const sweepEvaluatedAt = new Date() + const queuedGraceCutoff = new Date(sweepEvaluatedAt.getTime() - QUEUED_DISPATCH_GRACE_MS) + const processingStaleCutoff = new Date( + sweepEvaluatedAt.getTime() - STALE_PROCESSING_MINUTES * 60 * 1000 + ) + const sweepCandidates = await db + .select({ + id: document.id, + fileUrl: document.fileUrl, + filename: document.filename, + fileSize: document.fileSize, + mimeType: document.mimeType, + processingStatus: document.processingStatus, + processingQueuedAt: document.processingQueuedAt, + processingStartedAt: document.processingStartedAt, + processingDeferredUntil: document.processingDeferredUntil, + processingCompletedAt: document.processingCompletedAt, + uploadedAt: document.uploadedAt, + }) + .from(document) + .where( + and( + eq(document.connectorId, connectorId), + inArray(document.processingStatus, SWEEPABLE_PROCESSING_STATUSES), + or( + and( + eq(document.processingStatus, 'failed'), + sql`COALESCE(${document.processingCompletedAt}, ${document.processingQueuedAt}, ${document.uploadedAt}) < ${sql.param(queuedGraceCutoff, document.processingCompletedAt)}` + ), + and( + eq(document.processingStatus, 'pending'), + or( + and( + isNotNull(document.processingDeferredUntil), + lt(document.processingDeferredUntil, queuedGraceCutoff) + ), + and( + isNull(document.processingDeferredUntil), + sql`COALESCE(${document.processingQueuedAt}, ${document.uploadedAt}) < ${sql.param(queuedGraceCutoff, document.processingQueuedAt)}` + ) + ) + ), + and( + eq(document.processingStatus, 'processing'), + or( + isNull(document.processingStartedAt), + lt(document.processingStartedAt, processingStaleCutoff) + ) + ) + ), + // Dead letters are left alone: past the budget, re-dispatching only + // re-bills a document that has failed the same way every time. + lt(document.processingAttempts, MAX_PROCESSING_ATTEMPTS), + lt(document.uploadedAt, syncStartedAt), + gt(document.uploadedAt, retryCutoff), + eq(document.userExcluded, false), + isNotNull(document.storageKey), + isNull(document.archivedAt), + isNull(document.deletedAt) + ) + ) + .orderBy( + asc(sql`CASE + WHEN ${document.processingStatus} = 'failed' + THEN COALESCE(${document.processingCompletedAt}, ${document.processingQueuedAt}, ${document.uploadedAt}) + WHEN ${document.processingStatus} = 'pending' + THEN COALESCE(${document.processingDeferredUntil}, ${document.processingQueuedAt}, ${document.uploadedAt}) + ELSE COALESCE(${document.processingStartedAt}, ${sql.param(new Date(0), document.processingStartedAt)}) + END`), + asc(document.id) + ) + .limit(STUCK_RETRY_MAX_CANDIDATES_PER_SYNC) + const stuckDocs = sweepCandidates.filter( + (row): row is typeof row & { processingStatus: DocumentProcessingStatus } => + isDocumentProcessingStatus(row.processingStatus) + ) + + if (stuckDocs.length === 0) return + + logger.info(`Retrying ${stuckDocs.length} stuck documents`, { connectorId }) + try { + const stuckDocIds = stuckDocs.map((doc) => doc.id) + let retryDocs: typeof stuckDocs = [] + + /** + * Locks the parent first to match lifecycle mutations, then proves this + * run still owns the live connector row. A bare connector lock can match + * a replacement run after this lease was reclaimed, allowing the stale + * run to reset documents and dispatch duplicate processing. + */ + await db.transaction(async (tx) => { + const [activeKnowledgeBase] = await tx + .select({ id: knowledgeBase.id }) + .from(knowledgeBase) + .where(and(eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt))) + .for('update') + if (!activeKnowledgeBase) throw new SyncLockLostException(connectorId) + + const [heldSyncLock] = await tx + .select({ id: knowledgeConnector.id }) + .from(knowledgeConnector) + .where(input.lease.stillHeld()) + .for('update') + if (!heldSyncLock) throw new SyncLockLostException(connectorId) + + const lockedCandidates = await tx + .select({ + id: document.id, + fileUrl: document.fileUrl, + filename: document.filename, + fileSize: document.fileSize, + mimeType: document.mimeType, + processingStatus: document.processingStatus, + processingQueuedAt: document.processingQueuedAt, + processingStartedAt: document.processingStartedAt, + processingDeferredUntil: document.processingDeferredUntil, + processingCompletedAt: document.processingCompletedAt, + uploadedAt: document.uploadedAt, + }) + .from(document) + .where( + and( + inArray(document.id, stuckDocIds), + eq(document.connectorId, connectorId), + inArray(document.processingStatus, SWEEPABLE_PROCESSING_STATUSES), + lt(document.processingAttempts, MAX_PROCESSING_ATTEMPTS), + eq(document.userExcluded, false), + isNotNull(document.storageKey), + isNull(document.archivedAt), + isNull(document.deletedAt) + ) + ) + .orderBy(asc(document.id)) + .for('update') + + retryDocs = selectStuckDocumentSweepCandidates( + lockedCandidates.filter( + (row): row is typeof row & { processingStatus: DocumentProcessingStatus } => + isDocumentProcessingStatus(row.processingStatus) + ), + sweepEvaluatedAt + ) + + if (retryDocs.length > 0) { + const retryDocIds = retryDocs.map((doc) => doc.id) + + const reset = await tx + .update(document) + .set({ + processingStatus: 'pending', + /** + * Invalidates the prior dispatch generation in the same write + * that reopens the row. The dispatch below installs its fresh + * generation through `markDocumentsQueued`. + */ + processingQueuedAt: null, + processingQueueToken: null, + processingStartedAt: null, + processingDeferredUntil: null, + processingCompletedAt: null, + processingError: null, + chunkCount: 0, + tokenCount: 0, + characterCount: 0, + }) + /** + * These rows were freshly revalidated and locked above. The + * lifecycle predicates remain as defence in depth; the row locks + * ensure no retry can install a newer queue generation between + * that eligibility decision and this reset. + */ + .where( + and( + inArray(document.id, retryDocIds), + eq(document.connectorId, connectorId), + inArray(document.processingStatus, SWEEPABLE_PROCESSING_STATUSES), + lt(document.processingAttempts, MAX_PROCESSING_ATTEMPTS), + eq(document.userExcluded, false), + isNotNull(document.storageKey), + isNull(document.archivedAt), + isNull(document.deletedAt) + ) + ) + .returning({ id: document.id }) + + // Embeddings are dropped only for documents this sweep actually + // reset. Deleting first would strip a pass that completed between + // the candidate SELECT and this write. + const resetIds = reset.map((row) => row.id) + if (resetIds.length > 0) { + await tx.delete(embedding).where(inArray(embedding.documentId, resetIds)) + } + const resetIdSet = new Set(resetIds) + retryDocs = retryDocs.filter((doc) => resetIdSet.has(doc.id)) + } + }) + + for (let i = 0; i < retryDocs.length; i += STUCK_RETRY_DISPATCH_CHUNK_SIZE) { + await input.lease.beatLive() + + const retryChunk = retryDocs.slice(i, i + STUCK_RETRY_DISPATCH_CHUNK_SIZE) + result.processingDispatch.requested += retryChunk.length + const dispatch = await processDocumentsWithQueue( + retryChunk.map((doc) => ({ + documentId: doc.id, + filename: doc.filename ?? 'document.txt', + fileUrl: doc.fileUrl ?? '', + fileSize: doc.fileSize ?? 0, + mimeType: doc.mimeType ?? 'text/plain', + })), + knowledgeBaseId, + {}, + generateId(), + billingAttribution, + { connectorId, stillHeld: input.lease.stillHeld } + ) + result.processingDispatch.accepted += dispatch.accepted + result.processingDispatch.failed += dispatch.failed + } + } catch (error) { + /** + * Kept out of the best-effort swallow below. A run that has provably + * lost its lock would otherwise be mislabelled an enqueue failure, fall + * through and publish an atomic completed outcome, which a replacement + * run could then read as corroboration of its own listing. + */ + if (error instanceof SyncLockLostException) throw error + + logger.warn('Failed to enqueue stuck documents for reprocessing', { + connectorId, + count: stuckDocs.length, + error: toError(error).message, + }) + result.processingDispatch.failed += + result.processingDispatch.requested - + result.processingDispatch.accepted - + result.processingDispatch.failed + } +} diff --git a/apps/sim/lib/knowledge/documents/document-processing-source.test.ts b/apps/sim/lib/knowledge/documents/document-processing-source.test.ts index 96f2aee25ac..435c0c38116 100644 --- a/apps/sim/lib/knowledge/documents/document-processing-source.test.ts +++ b/apps/sim/lib/knowledge/documents/document-processing-source.test.ts @@ -78,6 +78,7 @@ import { EMBEDDING_QUOTA_EXHAUSTED_MESSAGE, } from '@/lib/embeddings' import { EmbeddingAPIError, EmbeddingQuotaExhaustedError } from '@/lib/embeddings/client' +import { SYSTEM_ACCESS_SCOPE } from '@/lib/knowledge/access/types' import { PermanentDocumentProcessingError, UsageLimitDocumentProcessingError, @@ -100,6 +101,7 @@ const PERSISTED_CONTEXT = { fileUrl: PERSISTED_URL, fileSize: 512, mimeType: 'application/pdf', + connectorId: null, tag1: null, tag2: null, tag3: null, @@ -217,7 +219,7 @@ describe('knowledge document processing source', () => { 1024, 200, 100, - PERSISTED_CONTEXT.uploadedBy, + { userId: PERSISTED_CONTEXT.uploadedBy, knowledgeAccess: undefined }, null, undefined, undefined @@ -225,6 +227,35 @@ describe('knowledge document processing source', () => { expect(mockGenerateEmbeddings).not.toHaveBeenCalled() }) + it('reads a connector-owned source file as the system, not as the actor', async () => { + resetDbChainMock() + dbChainMockFns.returning.mockResolvedValue([{ id: 'document-1' }]) + dbChainMockFns.limit + .mockResolvedValueOnce([{ ...PERSISTED_CONTEXT, connectorId: 'connector-1' }]) + .mockResolvedValueOnce([PERSISTED_PROVENANCE_ROW]) + .mockResolvedValueOnce([{ id: 'document-1' }]) + + await processDocumentAsync('knowledge-base-1', 'document-1', { + filename: PERSISTED_CONTEXT.filename, + fileUrl: PERSISTED_CONTEXT.fileUrl, + fileSize: PERSISTED_CONTEXT.fileSize, + mimeType: PERSISTED_CONTEXT.mimeType, + }) + + expect(mockProcessDocument).toHaveBeenCalledWith( + PERSISTED_CONTEXT.fileUrl, + PERSISTED_CONTEXT.filename, + PERSISTED_CONTEXT.mimeType, + 1024, + 200, + 100, + { userId: PERSISTED_CONTEXT.uploadedBy, knowledgeAccess: SYSTEM_ACCESS_SCOPE }, + null, + undefined, + undefined + ) + }) + it('processes a legacy document when its workspace metadata row no longer exists', async () => { mockGetFileMetadataByKeys.mockResolvedValue([]) mockGetBoundWorkspaceFileSecretProvenanceByMetadata.mockResolvedValue(new Map()) @@ -243,7 +274,7 @@ describe('knowledge document processing source', () => { 1024, 200, 100, - PERSISTED_CONTEXT.uploadedBy, + { userId: PERSISTED_CONTEXT.uploadedBy, knowledgeAccess: undefined }, null, undefined, undefined diff --git a/apps/sim/lib/knowledge/documents/document-processor.ts b/apps/sim/lib/knowledge/documents/document-processor.ts index ed486c519a6..a1a48008db5 100644 --- a/apps/sim/lib/knowledge/documents/document-processor.ts +++ b/apps/sim/lib/knowledge/documents/document-processor.ts @@ -60,7 +60,10 @@ import { import { StorageService } from '@/lib/uploads' import { buildStorageKeySegment } from '@/lib/uploads/core/storage-key' import { getFileExtension, isInternalFileUrl } from '@/lib/uploads/utils/file-utils' -import { downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server' +import { + type DownloadFileFromUrlOptions, + downloadFileFromUrl, +} from '@/lib/uploads/utils/file-utils.server' import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation' import { mistralParserTool } from '@/tools/mistral/parser' @@ -194,6 +197,12 @@ async function applyStrategy( } } +/** + * Who a source-file read runs as: the actor for authorization and OCR + * attribution, plus how a knowledge-base file identifies its reader. + */ +export type SourceFileAccess = Pick + export async function processDocument( fileUrl: string, filename: string, @@ -201,7 +210,7 @@ export async function processDocument( chunkSize = 1024, chunkOverlap = 200, minCharactersPerChunk = 100, - userId?: string, + access: SourceFileAccess = {}, workspaceId?: string | null, strategy?: ChunkingStrategy, strategyOptions?: StrategyOptions @@ -221,7 +230,7 @@ export async function processDocument( logger.info('Processing document', { mimeType }) try { - const parseResult = await parseDocument(fileUrl, filename, mimeType, userId, workspaceId) + const parseResult = await parseDocument(fileUrl, filename, mimeType, access, workspaceId) const { content, processingMethod } = parseResult const cloudUrl = 'cloudUrl' in parseResult ? parseResult.cloudUrl : undefined @@ -355,7 +364,7 @@ async function readEmbeddedPdfText( fileUrl: string, filename: string, mimeType: string, - userId?: string + access: SourceFileAccess ): Promise< | { content: string @@ -366,7 +375,7 @@ async function readEmbeddedPdfText( | undefined > { try { - const buffer = await downloadFileWithTimeout(fileUrl, userId) + const buffer = await downloadFileWithTimeout(fileUrl, access) const parsed = await parseBuffer(buffer, 'pdf') /** @@ -408,7 +417,7 @@ async function parseDocument( fileUrl: string, filename: string, mimeType: string, - userId?: string, + access: SourceFileAccess, workspaceId?: string | null ): Promise<{ content: string @@ -435,30 +444,30 @@ async function parseDocument( * documents that actually need it — which also means everything else stops * depending on that service being reachable. */ - const embedded = await readEmbeddedPdfText(fileUrl, filename, mimeType, userId) + const embedded = await readEmbeddedPdfText(fileUrl, filename, mimeType, access) if (embedded) return embedded assertKnowledgeOpaqueModelInputSafe() if (ocrProvider === 'azure-mistral') { logger.info('Using Azure Mistral OCR') - return parseWithAzureMistralOCR(fileUrl, filename, mimeType, userId) + return parseWithAzureMistralOCR(fileUrl, filename, mimeType, access) } logger.info('Using Mistral OCR') - return parseWithMistralOCR(fileUrl, filename, mimeType, userId, workspaceId, mistralApiKey) + return parseWithMistralOCR(fileUrl, filename, mimeType, access, workspaceId, mistralApiKey) } } logger.info('Using file parser') - return parseWithFileParser(fileUrl, filename, mimeType, userId) + return parseWithFileParser(fileUrl, filename, mimeType, access) } async function handleFileForOCR( fileUrl: string, filename: string, mimeType: string, - userId?: string, + access: SourceFileAccess, workspaceId?: string | null ) { const isExternalHttps = /^https:\/\//i.test(fileUrl) && !isInternalFileUrl(fileUrl) @@ -466,7 +475,7 @@ async function handleFileForOCR( if (isExternalHttps) { if (mimeType === 'application/pdf') { logger.info('handleFileForOCR: Downloading external PDF for OCR admission') - const buffer = await downloadFileWithTimeout(fileUrl, userId) + const buffer = await downloadFileWithTimeout(fileUrl, access) logger.info('handleFileForOCR: Downloaded external PDF', { bytes: buffer.length }) return { httpsUrl: fileUrl, buffer } } @@ -476,7 +485,7 @@ async function handleFileForOCR( logger.info('Uploading document to cloud storage for OCR') - const buffer = await downloadFileWithTimeout(fileUrl, userId) + const buffer = await downloadFileWithTimeout(fileUrl, access) logger.info('Downloaded document for OCR', { bytes: buffer.length }) @@ -485,7 +494,7 @@ async function handleFileForOCR( originalName: filename, uploadedAt: new Date().toISOString(), purpose: 'knowledge-base', - ...(userId && { userId }), + ...(access.userId && { userId: access.userId }), ...(workspaceId && { workspaceId }), } @@ -521,20 +530,20 @@ async function handleFileForOCR( * up front on an oversized `Content-Length`), so an attacker-controlled `fileUrl` * pointing at an unbounded body cannot exhaust the processing worker's memory. */ -async function downloadFileWithTimeout(fileUrl: string, userId?: string): Promise { +async function downloadFileWithTimeout(fileUrl: string, access: SourceFileAccess): Promise { return downloadFileFromUrl(fileUrl, { timeoutMs: TIMEOUTS.FILE_DOWNLOAD, maxBytes: MAX_FILE_SIZE, - userId, + ...access, }) } -async function downloadFileForBase64(fileUrl: string, userId?: string): Promise { +async function downloadFileForBase64(fileUrl: string, access: SourceFileAccess): Promise { if (/^data:/i.test(fileUrl)) { return decodeDataUriWithinLimit(fileUrl, MAX_FILE_SIZE).buffer } if (/^https?:\/\//i.test(fileUrl) || isInternalFileUrl(fileUrl)) { - return downloadFileWithTimeout(fileUrl, userId) + return downloadFileWithTimeout(fileUrl, access) } throw new Error( 'Unsupported fileUrl scheme: only data: URIs, http(s):// URLs, and internal /api/files/serve/ paths are allowed' @@ -678,7 +687,7 @@ async function parseWithAzureMistralOCR( fileUrl: string, filename: string, mimeType: string, - userId?: string + access: SourceFileAccess ) { validateOCRConfig( env.OCR_AZURE_API_KEY, @@ -687,7 +696,7 @@ async function parseWithAzureMistralOCR( 'Azure Mistral OCR' ) - const fileBuffer = await downloadFileForBase64(fileUrl, userId) + const fileBuffer = await downloadFileForBase64(fileUrl, access) const requestPolicy = getAzureMistralOcrRequestPolicy(env.OCR_AZURE_MODEL_NAME!) try { @@ -792,7 +801,7 @@ async function parseWithMistralOCR( fileUrl: string, filename: string, mimeType: string, - userId?: string, + access: SourceFileAccess, workspaceId?: string | null, mistralApiKey?: string | null ) { @@ -805,7 +814,7 @@ async function parseWithMistralOCR( fileUrl, filename, mimeType, - userId, + access, workspaceId ) @@ -829,13 +838,13 @@ async function parseWithMistralOCR( maxBytes: MISTRAL_OCR_REQUEST_POLICY.maxBytes, maxPages: MISTRAL_OCR_REQUEST_POLICY.maxPages, }) - return processMistralOCRInBatches(filename, apiKey, buffer, userId, cloudUrl) + return processMistralOCRInBatches(filename, apiKey, buffer, access, cloudUrl) } const params = { filePath: httpsUrl, apiKey, resultType: 'text' as const } try { - const response = await executeMistralOCRRequest(params, userId) + const response = await executeMistralOCRRequest(params, access) const result = (await mistralParserTool.transformResponse!(response, params)) as OCRResult const content = processOCRContent(result, filename, pageCount > 0 ? pageCount : undefined) @@ -850,7 +859,7 @@ async function parseWithMistralOCR( async function executeMistralOCRRequest( params: { filePath: string; apiKey: string; resultType: 'text' }, - userId?: string + access: SourceFileAccess ): Promise { return retryWithExponentialBackoff( async () => { @@ -876,7 +885,7 @@ async function executeMistralOCRRequest( requestId: generateId(), signal: controller.signal, trustedCaller: 'knowledge-ingestion', - userId, + userId: access.userId, }) return Response.json(result) } catch (error) { @@ -905,7 +914,7 @@ async function processChunk( chunkIndex: number, filename: string, apiKey: string, - userId?: string + access: SourceFileAccess ): Promise { const chunkPageCount = chunk.endPage - chunk.startPage + 1 @@ -955,7 +964,7 @@ async function processChunk( resultType: 'text' as const, } - const response = await executeMistralOCRRequest(params, userId) + const response = await executeMistralOCRRequest(params, access) const result = (await mistralParserTool.transformResponse!(response, params)) as OCRResult if (!result.success) { @@ -1192,7 +1201,7 @@ async function processMistralOCRInBatches( filename: string, apiKey: string, pdfBuffer: Buffer, - userId?: string, + access: SourceFileAccess, cloudUrl?: string ): Promise<{ content: string @@ -1204,7 +1213,7 @@ async function processMistralOCRInBatches( 'mistral', filename, MISTRAL_OCR_REQUEST_POLICY, - (chunk, index) => processChunk(chunk, index, filename, apiKey, userId) + (chunk, index) => processChunk(chunk, index, filename, apiKey, access) ) return { content, processingMethod: 'mistral-ocr', cloudUrl } @@ -1231,7 +1240,7 @@ async function parseWithFileParser( fileUrl: string, filename: string, mimeType: string, - userId?: string + access: SourceFileAccess ) { try { let content: string @@ -1245,7 +1254,7 @@ async function parseWithFileParser( // Internal URLs may arrive as an app-relative `/api/files/serve/...` path // (some ingestion callers store the relative path); downloadFileFromUrl // resolves it directly against storage without an absolute origin. - const result = await parseHttpFile(fileUrl, filename, mimeType, userId) + const result = await parseHttpFile(fileUrl, filename, mimeType, access) content = result.content metadata = result.metadata || {} } else { @@ -1275,10 +1284,10 @@ async function parseDataURI( async function parseHttpFile( fileUrl: string, filename: string, - mimeType?: string, - userId?: string + mimeType: string | undefined, + access: SourceFileAccess ): Promise<{ content: string; metadata?: FileParseMetadata }> { - const buffer = await downloadFileWithTimeout(fileUrl, userId) + const buffer = await downloadFileWithTimeout(fileUrl, access) /** Prefer what we actually downloaded over what the document is *called*. */ const extension = diff --git a/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts b/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts index b450f60897d..e8dc4c4bbde 100644 --- a/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts +++ b/apps/sim/lib/knowledge/documents/pdf-ocr-triage.test.ts @@ -57,7 +57,10 @@ function ocrPages(count: number, markdown = 'Recognised page') { function parse() { return runWithKnowledgeModelInputProvenance( undefined, - () => processDocument(PDF_URL, 'Contract.pdf', 'application/pdf', 1024, 200, 1, 'user-1'), + () => + processDocument(PDF_URL, 'Contract.pdf', 'application/pdf', 1024, 200, 1, { + userId: 'user-1', + }), { opaqueInputSafe: true } ) } diff --git a/apps/sim/lib/knowledge/documents/processing-outbox-handler.test.ts b/apps/sim/lib/knowledge/documents/processing-outbox-handler.test.ts index 343402fa360..9df3472ac6d 100644 --- a/apps/sim/lib/knowledge/documents/processing-outbox-handler.test.ts +++ b/apps/sim/lib/knowledge/documents/processing-outbox-handler.test.ts @@ -21,6 +21,7 @@ vi.mock('@/lib/knowledge/documents/processing-claim', () => ({ import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' import type { OutboxEventContext } from '@/lib/core/outbox/service' +import { SYSTEM_ACCESS_SCOPE } from '@/lib/knowledge/access/types' import { KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT } from '@/lib/knowledge/documents/processing-outbox-event' import { knowledgeDocumentProcessingOutboxHandlers } from '@/lib/knowledge/documents/processing-outbox-handler' @@ -87,7 +88,11 @@ describe('knowledge document processing outbox handler', () => { it('dispatches the authoritative document with the stable outbox event id', async () => { await handler()(PAYLOAD, createContext('outbox-event-stable')) - expect(mocks.getKnowledgeDocument).toHaveBeenCalledWith('knowledge-base-1', 'document-1') + expect(mocks.getKnowledgeDocument).toHaveBeenCalledWith( + 'knowledge-base-1', + 'document-1', + SYSTEM_ACCESS_SCOPE + ) expect(mocks.processDocumentsWithQueue).toHaveBeenCalledWith( [ { diff --git a/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts b/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts index c008680600c..220f932f7db 100644 --- a/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts +++ b/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts @@ -1,5 +1,6 @@ import { assertBillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' import type { OutboxHandler, OutboxHandlerRegistry } from '@/lib/core/outbox/service' +import { SYSTEM_ACCESS_SCOPE } from '@/lib/knowledge/access/types' import { reclaimStaleDocumentProcessingClaim } from '@/lib/knowledge/documents/processing-claim' import { KNOWLEDGE_DOCUMENT_PROCESSING_OUTBOX_EVENT, @@ -58,7 +59,12 @@ function parsePayload(payload: unknown): KnowledgeDocumentProcessingOutboxPayloa const processKnowledgeDocument: OutboxHandler = async (rawPayload, context) => { const payload = parsePayload(rawPayload) context.signal.throwIfAborted() - const document = await getKnowledgeDocument(payload.knowledgeBaseId, payload.documentId) + /** A background job processing the row it was dispatched for; no principal is involved. */ + const document = await getKnowledgeDocument( + payload.knowledgeBaseId, + payload.documentId, + SYSTEM_ACCESS_SCOPE + ) if (!document || document.processingStatus === 'completed') return if (document.processingStatus === 'processing') { const reclaimed = await reclaimStaleDocumentProcessingClaim({ diff --git a/apps/sim/lib/knowledge/documents/processing-queue.test.ts b/apps/sim/lib/knowledge/documents/processing-queue.test.ts index 9e5fe8dda47..f5da3cbfafc 100644 --- a/apps/sim/lib/knowledge/documents/processing-queue.test.ts +++ b/apps/sim/lib/knowledge/documents/processing-queue.test.ts @@ -20,6 +20,7 @@ import { markInsideTriggerRun, resetInsideTriggerRunForTests, } from '@/lib/core/config/trigger-runtime' +import { SyncLockLostException } from '@/lib/knowledge/connectors/sync-lock' import { DOCUMENT_PROCESSING_STALE_THRESHOLD_MS } from '@/lib/knowledge/documents/processing-timeouts.server' import { QUEUED_DISPATCH_GRACE_MS } from '@/lib/knowledge/documents/types' @@ -1060,3 +1061,68 @@ describe('processDocumentsWithQueue attempt refund', () => { ).toBe(false) }) }) + +describe('processDocumentsWithQueue under a connector sync lease', () => { + const lease = { + connectorId: 'connector-1', + stillHeld: () => ({ type: 'lease' }) as never, + } + + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mockBatchTrigger.mockResolvedValue({ batchId: 'batch-1' }) + mockResolveTriggerRegion.mockResolvedValue('us-east-1') + for (const key of Object.keys(env)) { + delete (env as Record)[key] + } + Object.assign(env, { ...defaultMockEnv, TRIGGER_SECRET_KEY: 'trigger-secret' }) + }) + + /** + * The document writes proved the lease in their own transactions; the queue + * write is a later one. A run reclaimed in between must not install a + * processing generation, spend an attempt, or dispatch beside the + * replacement run's own dispatch for the same document. + */ + it('neither marks nor dispatches processing once the lease was reclaimed', async () => { + dbChainMockFns.for.mockResolvedValueOnce([]) + + await expect( + processDocumentsWithQueue( + [DOCUMENT], + 'knowledge-base-1', + {}, + 'request-1', + BILLING_ATTRIBUTION, + lease + ) + ).rejects.toBeInstanceOf(SyncLockLostException) + + expect(dbChainMockFns.where).toHaveBeenCalledWith(lease.stillHeld()) + expect(dbChainMockFns.update).not.toHaveBeenCalled() + expect(mockBatchTrigger).not.toHaveBeenCalled() + }) + + it('queues and dispatches while the lease is still held', async () => { + dbChainMockFns.for.mockResolvedValueOnce([{ id: 'connector-1' }]) + dbChainMockFns.returning.mockResolvedValue([{ id: 'document-1' }]) + dbChainMockFns.limit.mockResolvedValueOnce([ + { userId: 'knowledge-owner', workspaceId: 'workspace-1' }, + ]) + + await expect( + processDocumentsWithQueue( + [DOCUMENT], + 'knowledge-base-1', + {}, + 'request-1', + BILLING_ATTRIBUTION, + lease + ) + ).resolves.toEqual({ requested: 1, accepted: 1, failed: 0, failedDocumentIds: [] }) + + expect(dbChainMockFns.where).toHaveBeenCalledWith(lease.stillHeld()) + expect(mockBatchTrigger).toHaveBeenCalledTimes(1) + }) +}) diff --git a/apps/sim/lib/knowledge/documents/retry-processing-grace.test.ts b/apps/sim/lib/knowledge/documents/retry-processing-grace.test.ts index d7cb8abf547..34cbcc84dc5 100644 --- a/apps/sim/lib/knowledge/documents/retry-processing-grace.test.ts +++ b/apps/sim/lib/knowledge/documents/retry-processing-grace.test.ts @@ -18,7 +18,7 @@ vi.mock('@/lib/knowledge/documents/processing-outbox-event', () => ({ vi.mock('@/lib/uploads', () => ({ StorageService: {} })) vi.mock('@/connectors/registry.server', () => ({ CONNECTOR_REGISTRY: {} })) -import { isStuckDocumentSweepEligible } from '@/lib/knowledge/connectors/sync-engine' +import { isStuckDocumentSweepEligible } from '@/lib/knowledge/connectors/sync-primitives' import { processDocumentsWithQueue, retryDocumentProcessing, diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index 164d71b04a2..cab2ea2f4f7 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -76,6 +76,13 @@ import { EXACT_EMPTY_DURABLE_SECRET_PROVENANCE, mergeDurableSecretProvenance, } from '@/lib/execution/durable-secret-provenance' +import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' +import { + type KnowledgeAccessScope, + SYSTEM_ACCESS_SCOPE, + type SystemAccessScope, +} from '@/lib/knowledge/access/types' +import { assertSyncLeaseHeldInTx, type SyncWriteLease } from '@/lib/knowledge/connectors/sync-lock' import { assertDocumentChunkCountWithinLimit, isPermanentDocumentProcessingError, @@ -84,7 +91,10 @@ import { toPermanentDocumentProcessingError, UsageLimitDocumentProcessingError, } from '@/lib/knowledge/documents/document-processing-error' -import { processDocument } from '@/lib/knowledge/documents/document-processor' +import { + processDocument, + type SourceFileAccess, +} from '@/lib/knowledge/documents/document-processor' import { failStaleDocumentProcessingClaim, recordUndispatchedDocumentFailure, @@ -811,14 +821,27 @@ async function isDocumentAcceptedWithoutDispatch( return accepted.length > 0 } +/** + * The sync run a connector dispatch proves before it queues processing. The + * document writes prove the lease in their own transactions, but the queue + * write is a later transaction: a run reclaimed in between would otherwise + * install a processing generation, spend an attempt, and dispatch a worker + * beside the replacement run's own dispatch for the same document. + */ +export interface ProcessingDispatchLease extends SyncWriteLease { + connectorId: string +} + async function markDocumentsQueued( documentIds: string[], knowledgeBaseId: string, queueToken: string, - queuedAt: Date + queuedAt: Date, + lease: ProcessingDispatchLease | undefined ): Promise { const legacyAdoptionCutoff = new Date(queuedAt.getTime() - QUEUED_DISPATCH_GRACE_MS) return db.transaction(async (tx) => { + if (lease) await assertSyncLeaseHeldInTx(tx, lease.connectorId, lease) const claimed = await tx .update(document) .set({ @@ -1007,14 +1030,16 @@ async function bestEffortWithdrawDocumentsQueued( * available, or in-process otherwise. Throws only when every dispatch fails; * partial failures are returned and recovered by the next sync's stuck-doc * pass. A successful Trigger.dev hand-off is only an accepted child run, not a - * claim about its eventual processing outcome. + * claim about its eventual processing outcome. A connector sync passes its + * lease, and the queue write then lands only while the run still holds it. */ export async function processDocumentsWithQueue( createdDocuments: DocumentData[], knowledgeBaseId: string, processingOptions: ProcessingOptions, requestId: string, - billingAttribution: BillingAttributionSnapshot | undefined + billingAttribution: BillingAttributionSnapshot | undefined, + lease?: ProcessingDispatchLease ): Promise { const seenDocumentIds = new Set() const uniqueDocuments = createdDocuments.filter((createdDocument) => { @@ -1033,7 +1058,7 @@ export async function processDocumentsWithQueue( generations: queuedGenerations, acceptedWithoutDispatchIds, unresolvedIds, - } = await markDocumentsQueued(documentIds, knowledgeBaseId, requestId, queuedAt) + } = await markDocumentsQueued(documentIds, knowledgeBaseId, requestId, queuedAt, lease) const generationByDocumentId = new Map( queuedGenerations.map((generation) => [generation.documentId, generation]) ) @@ -1325,6 +1350,18 @@ function queueGenerationConditions( * invocation against the document's retry budget. Direct callers omit it and * therefore cannot refund an attempt they never charged. */ +/** + * Who the processor reads a document's source file as. Always the actor, not + * the payer: authorizing as the KB owner would let a writer ingest an internal + * file only the owner can read. A connector-owned row was written by the sync + * from bytes it fetched, not from a caller-supplied URL, so it is read as the + * system: in members mode the row stays hidden until the sync materializes who + * observed it, and the actor's own scope would deny the read. + */ +function sourceFileAccessFor(connectorId: string | null, actorUserId: string): SourceFileAccess { + return { userId: actorUserId, knowledgeAccess: connectorId ? SYSTEM_ACCESS_SCOPE : undefined } +} + export async function processDocumentAsync( knowledgeBaseId: string, documentId: string, @@ -1358,6 +1395,7 @@ export async function processDocumentAsync( embeddingModel: knowledgeBase.embeddingModel, billedAccountUserId: workspaceTable.billedAccountUserId, uploadedBy: document.uploadedBy, + connectorId: document.connectorId, filename: document.filename, fileUrl: document.fileUrl, fileSize: document.fileSize, @@ -1568,12 +1606,7 @@ export async function processDocumentAsync( kbConfig.maxSize, kbConfig.overlap, kbConfig.minSize, - /** - * Authorize source-file processing as the actor, not the payer. Using - * the KB owner would let a writer ingest an internal file that only the - * owner can read. - */ - documentActorUserId, + sourceFileAccessFor(ctx.connectorId, documentActorUserId), ctx.workspaceId, rawConfig?.strategy, rawConfig?.strategyOptions @@ -2341,7 +2374,8 @@ export async function getDocuments( sortOrder?: SortOrder tagFilters?: TagFilterCondition[] }, - requestId: string + requestId: string, + access: KnowledgeAccessScope | SystemAccessScope ): Promise<{ documents: Array<{ id: string @@ -2402,6 +2436,7 @@ export async function getDocuments( eq(document.userExcluded, false), isNull(document.archivedAt), isNull(document.deletedAt), + knowledgeAccessCondition(access), ] if (enabledFilter === 'enabled') { @@ -2555,10 +2590,15 @@ export type ActiveKnowledgeDocument = typeof document.$inferSelect & { connectorType: string | null } -/** Loads one visible document and its connector metadata for every API adapter. */ +/** + * Loads one visible document and its connector metadata for every API adapter. + * A document the caller may not read is reported as absent, the same as one + * that does not exist, so no surface can confirm a restricted document exists. + */ export async function getKnowledgeDocument( knowledgeBaseId: string, - documentId: string + documentId: string, + access: KnowledgeAccessScope | SystemAccessScope ): Promise { const [row] = await db .select({ @@ -2573,7 +2613,8 @@ export async function getKnowledgeDocument( eq(document.knowledgeBaseId, knowledgeBaseId), eq(document.userExcluded, false), isNull(document.archivedAt), - isNull(document.deletedAt) + isNull(document.deletedAt), + knowledgeAccessCondition(access) ) ) .limit(1) @@ -2583,7 +2624,8 @@ export async function getKnowledgeDocument( /** Loads one visible document by its canonical ID before any asserted parent is trusted. */ export async function getKnowledgeDocumentById( - documentId: string + documentId: string, + access: KnowledgeAccessScope | SystemAccessScope ): Promise { const [row] = await db .select({ @@ -2597,7 +2639,8 @@ export async function getKnowledgeDocumentById( eq(document.id, documentId), eq(document.userExcluded, false), isNull(document.archivedAt), - isNull(document.deletedAt) + isNull(document.deletedAt), + knowledgeAccessCondition(access) ) ) .limit(1) @@ -2933,6 +2976,7 @@ export async function bulkDocumentOperation( knowledgeBaseId: string, operation: 'enable' | 'disable' | 'delete', documentIds: string[], + access: KnowledgeAccessScope, requestId: string ): Promise<{ success: boolean @@ -2960,7 +3004,8 @@ export async function bulkDocumentOperation( inArray(document.id, documentIds), eq(document.userExcluded, false), isNull(document.archivedAt), - isNull(document.deletedAt) + isNull(document.deletedAt), + knowledgeAccessCondition(access) ) ) @@ -2996,7 +3041,10 @@ export async function bulkDocumentOperation( .where( and( eq(document.knowledgeBaseId, knowledgeBaseId), - inArray(document.id, documentIds), + inArray( + document.id, + documentsToUpdate.map((doc) => doc.id) + ), eq(document.userExcluded, false), isNull(document.archivedAt), isNull(document.deletedAt) @@ -3022,6 +3070,7 @@ export async function bulkDocumentOperationByFilter( knowledgeBaseId: string, operation: 'enable' | 'disable' | 'delete', enabledFilter: 'all' | 'enabled' | 'disabled' | undefined, + access: KnowledgeAccessScope, requestId: string ): Promise<{ success: boolean @@ -3041,6 +3090,8 @@ export async function bulkDocumentOperationByFilter( eq(document.userExcluded, false), isNull(document.archivedAt), isNull(document.deletedAt), + /** "Every document" means every document the caller can see. */ + knowledgeAccessCondition(access), ] if (enabledFilter === 'enabled') { @@ -3636,10 +3687,17 @@ async function excludeConnectorDocuments( return updated.length } +/** + * Deletes documents by their lifecycle: connector-owned ones are excluded, + * uploads are hard deleted. `access`, when given, is applied to the selection + * and to every write, so a document the caller stopped being able to read + * after they looked it up is left alone rather than deleted on a stale view. + */ async function deleteDocumentsByLifecyclePolicy( documentIds: string[], requestId: string, - expectedKnowledgeBaseId?: string + expectedKnowledgeBaseId?: string, + access?: KnowledgeAccessScope ): Promise { const ids = [...new Set(documentIds)] if (ids.length === 0) { @@ -3659,7 +3717,8 @@ async function deleteDocumentsByLifecyclePolicy( eq(document.knowledgeBaseId, expectedKnowledgeBaseId), eq(document.userExcluded, false), isNull(document.archivedAt), - isNull(document.deletedAt) + isNull(document.deletedAt), + access ? knowledgeAccessCondition(access) : undefined ) : inArray(document.id, ids) ) @@ -3669,9 +3728,21 @@ async function deleteDocumentsByLifecyclePolicy( const [excludedCount, hardDeletedCount] = await Promise.all([ expectedKnowledgeBaseId - ? excludeConnectorKnowledgeDocuments(expectedKnowledgeBaseId, connectorBackedIds, requestId) + ? excludeConnectorKnowledgeDocuments( + expectedKnowledgeBaseId, + connectorBackedIds, + requestId, + access + ) : excludeConnectorDocuments(connectorBackedIds, requestId), - hardDeleteDocuments(hardDeleteIds, requestId, undefined, expectedKnowledgeBaseId), + hardDeleteDocuments( + hardDeleteIds, + requestId, + undefined, + expectedKnowledgeBaseId, + undefined, + access + ), ]) return excludedCount + hardDeletedCount @@ -3688,6 +3759,25 @@ export interface ConnectorSyncDeletionGuard { connectorId: string knowledgeBaseId: string syncLockToken: string + /** + * Which engine's lease the token belongs to. The content engine locks + * `sync_lock_token`; the members-mode engine locks `member_sync_lock_token`, + * and the two never coexist on one connector. + */ + lease?: 'content' | 'member' +} + +/** The lease predicate a deletion guard re-verifies under `FOR UPDATE`. */ +function connectorSyncGuardHeld(guard: ConnectorSyncDeletionGuard) { + return guard.lease === 'member' + ? and( + eq(knowledgeConnector.memberSyncStatus, 'running'), + eq(knowledgeConnector.memberSyncLockToken, guard.syncLockToken) + ) + : and( + eq(knowledgeConnector.status, 'syncing'), + eq(knowledgeConnector.syncLockToken, guard.syncLockToken) + ) } export async function hardDeleteDocuments( @@ -3704,7 +3794,9 @@ export async function hardDeleteDocuments( */ expectedConnectorId?: string, expectedKnowledgeBaseId?: string, - connectorSyncGuard?: ConnectorSyncDeletionGuard + connectorSyncGuard?: ConnectorSyncDeletionGuard, + /** When provided, only documents the caller may currently read are deleted, re-verified at the delete itself. */ + access?: KnowledgeAccessScope ): Promise { const ids = [...new Set(documentIds)] if (ids.length === 0) { @@ -3718,7 +3810,8 @@ export async function hardDeleteDocuments( requestId, expectedConnectorId, expectedKnowledgeBaseId, - connectorSyncGuard + connectorSyncGuard, + access ) } return deletedCount @@ -3733,13 +3826,15 @@ async function hardDeleteDocumentBatch( requestId: string, expectedConnectorId?: string, expectedKnowledgeBaseId?: string, - connectorSyncGuard?: ConnectorSyncDeletionGuard + connectorSyncGuard?: ConnectorSyncDeletionGuard, + access?: KnowledgeAccessScope ): Promise { const ids = [...new Set(documentIds)] const scopedConnectorId = connectorSyncGuard?.connectorId ?? expectedConnectorId const scopedKnowledgeBaseId = connectorSyncGuard?.knowledgeBaseId ?? expectedKnowledgeBaseId const requireEligibleDocument = Boolean(expectedKnowledgeBaseId || connectorSyncGuard) const requireVisibleDocument = Boolean(expectedKnowledgeBaseId && !connectorSyncGuard) + const accessCondition = access ? knowledgeAccessCondition(access) : undefined const documentsToDelete = await db .select({ id: document.id, @@ -3760,7 +3855,8 @@ async function hardDeleteDocumentBatch( scopedKnowledgeBaseId ? eq(document.knowledgeBaseId, scopedKnowledgeBaseId) : undefined, requireEligibleDocument ? eq(document.userExcluded, false) : undefined, requireEligibleDocument ? isNull(document.archivedAt) : undefined, - requireVisibleDocument ? isNull(document.deletedAt) : undefined + requireVisibleDocument ? isNull(document.deletedAt) : undefined, + accessCondition ) ) @@ -3849,8 +3945,7 @@ async function hardDeleteDocumentBatch( and( eq(knowledgeConnector.id, connectorSyncGuard.connectorId), eq(knowledgeConnector.knowledgeBaseId, connectorSyncGuard.knowledgeBaseId), - eq(knowledgeConnector.status, 'syncing'), - eq(knowledgeConnector.syncLockToken, connectorSyncGuard.syncLockToken), + connectorSyncGuardHeld(connectorSyncGuard), isNull(knowledgeConnector.archivedAt), isNull(knowledgeConnector.deletedAt) ) @@ -3873,7 +3968,7 @@ async function hardDeleteDocumentBatch( * ID set rather than the stale `existingIds`. */ const stillTargetedIds = - scopedConnectorId || scopedKnowledgeBaseId + scopedConnectorId || scopedKnowledgeBaseId || accessCondition ? ( await tx .select({ id: document.id }) @@ -3887,7 +3982,8 @@ async function hardDeleteDocumentBatch( : undefined, requireEligibleDocument ? eq(document.userExcluded, false) : undefined, requireEligibleDocument ? isNull(document.archivedAt) : undefined, - requireVisibleDocument ? isNull(document.deletedAt) : undefined + requireVisibleDocument ? isNull(document.deletedAt) : undefined, + accessCondition ) ) .orderBy(asc(document.id)) @@ -3960,22 +4056,33 @@ export async function deleteDocument( } } -/** Deletes one currently visible document within its canonical knowledge base. */ +/** + * Deletes one currently visible document within its canonical knowledge base. + * The caller's access is re-applied at the delete itself, so a token member + * sync revokes between the lookup and the write cannot still delete. + */ export async function deleteKnowledgeDocumentInKnowledgeBase( knowledgeBaseId: string, documentId: string, - requestId: string + requestId: string, + access: KnowledgeAccessScope ): Promise { - const current = await getKnowledgeDocument(knowledgeBaseId, documentId) + const current = await getKnowledgeDocument(knowledgeBaseId, documentId, access) if (!current) throw new OrchestrationError('not_found', 'Document not found') - const affected = await deleteDocumentsByLifecyclePolicy([documentId], requestId, knowledgeBaseId) + const affected = await deleteDocumentsByLifecyclePolicy( + [documentId], + requestId, + knowledgeBaseId, + access + ) if (affected !== 1) throw new OrchestrationError('not_found', 'Document not found') } async function excludeConnectorKnowledgeDocuments( knowledgeBaseId: string, documentIds: string[], - requestId: string + requestId: string, + access?: KnowledgeAccessScope ): Promise { if (documentIds.length === 0) return 0 const updated = await db @@ -3988,7 +4095,8 @@ async function excludeConnectorKnowledgeDocuments( isNotNull(document.connectorId), eq(document.userExcluded, false), isNull(document.archivedAt), - isNull(document.deletedAt) + isNull(document.deletedAt), + access ? knowledgeAccessCondition(access) : undefined ) ) .returning({ id: document.id }) diff --git a/apps/sim/lib/knowledge/documents/tag-filter.test.ts b/apps/sim/lib/knowledge/documents/tag-filter.test.ts index 94afd17c221..316a1a46b74 100644 --- a/apps/sim/lib/knowledge/documents/tag-filter.test.ts +++ b/apps/sim/lib/knowledge/documents/tag-filter.test.ts @@ -2,6 +2,7 @@ * @vitest-environment node */ import { describe, expect, it } from 'vitest' +import { WORKSPACE_ACCESS_SCOPE } from '@/lib/knowledge/access/scope' import { getDocuments } from '@/lib/knowledge/documents/service' import { buildTagFilterCondition } from '@/lib/knowledge/documents/tag-filter' import { validateTagValue } from '@/lib/knowledge/tags/utils' @@ -233,7 +234,8 @@ describe('getDocuments tag filters', () => { { tagSlot: 'not_a_real_slot', fieldType: 'text', operator: 'eq', value: 'x' }, ], }, - 'req-1' + 'req-1', + WORKSPACE_ACCESS_SCOPE ) ).rejects.toThrow(/Tag filter on slot "not_a_real_slot" could not be applied/) }) diff --git a/apps/sim/lib/knowledge/orchestration/connector-access.test.ts b/apps/sim/lib/knowledge/orchestration/connector-access.test.ts new file mode 100644 index 00000000000..58eabc3ae2e --- /dev/null +++ b/apps/sim/lib/knowledge/orchestration/connector-access.test.ts @@ -0,0 +1,561 @@ +/** + * @vitest-environment node + */ +import { + dbChainMockFns, + hasMockCondition, + type MockCondition, + queueTableRows, + resetDbChainMock, + schemaMock, +} from '@sim/testing' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + grant: vi.fn(), + revoke: vi.fn(), + validateBinding: vi.fn(), + loadGroup: vi.fn(), + dispatchSync: vi.fn(), + dispatchMemberSync: vi.fn(), + memberAccessAvailable: vi.fn(), + provision: vi.fn(), + rewriteAcls: vi.fn(), +})) + +vi.mock('@/lib/knowledge/connectors/member-observations', () => ({ + rewriteConnectorAcls: mocks.rewriteAcls, +})) + +vi.mock('@sim/audit', () => ({ + AuditAction: {}, + AuditResourceType: {}, + recordAudit: vi.fn(), +})) +vi.mock('@/lib/api-key/crypto', () => ({ encryptApiKey: vi.fn() })) +vi.mock('@/lib/billing/core/subscription', () => ({ hasWorkspaceLiveSyncAccess: vi.fn() })) +vi.mock('@/lib/knowledge/documents/service', () => ({ + deleteDocumentStorageFiles: vi.fn(), +})) +vi.mock('@/lib/knowledge/tags/service', () => ({ + cleanupUnusedTagDefinitions: vi.fn(), + createTagDefinition: vi.fn(), +})) +vi.mock('@/lib/posthog/server', () => ({ captureServerEvent: vi.fn() })) +vi.mock('@/lib/knowledge/connectors/member-access', () => ({ + grantKnowledgeConnectorCredentialAccess: mocks.grant, + revokeKnowledgeConnectorCredentialAccess: mocks.revoke, + validateKnowledgeConnectorMembersBinding: mocks.validateBinding, + findListingCapViolation: vi.fn(() => null), + stripListingCapFields: (_meta: unknown, sourceConfig: Record) => sourceConfig, +})) +vi.mock('@/lib/credential-groups/credentials', () => ({ + loadCredentialGroupCredentialListContext: mocks.loadGroup, +})) +vi.mock('@/lib/knowledge/access/availability', async () => { + const { OrchestrationError } = await import('@/lib/core/orchestration/types') + return { + isKnowledgeMemberAccessAvailable: mocks.memberAccessAvailable, + requireKnowledgeMemberAccessAvailable: async (context: { workspaceId: string }) => { + if (await mocks.memberAccessAvailable(context)) return + throw new OrchestrationError( + 'validation', + 'Per-member access is not available for this workspace' + ) + }, + } +}) +vi.mock('@/lib/knowledge/connectors/member-provisioning', () => ({ + provisionKnowledgeConnectorMembersBinding: mocks.provision, +})) +vi.mock('@/lib/knowledge/connectors/queue', () => ({ dispatchSync: mocks.dispatchSync })) +vi.mock('@/lib/knowledge/connectors/member-queue', () => ({ + dispatchMemberSync: mocks.dispatchMemberSync, +})) + +import { + performUpdateKnowledgeConnectorAccess, + resolveKnowledgeConnectorMembersBinding, +} from '@/lib/knowledge/orchestration/connector-access' + +const KB = { id: 'kb-1', name: 'Docs', workspaceId: 'ws-1' } +const ACTOR = { userId: 'admin-1', source: 'ui' as const, requestId: 'req-1' } +const BILLING = { actorUserId: 'admin-1', workspaceId: 'ws-1' } as never +const resolveBillingAttribution = vi.fn().mockResolvedValue(BILLING) + +const WORKSPACE_CONNECTOR = { + id: 'c-1', + knowledgeBaseId: 'kb-1', + connectorType: 'google_drive', + credentialId: 'cred-1', + encryptedApiKey: null, + sourceConfig: { folderId: ['f-1'] }, + syncMode: 'full', + syncIntervalMinutes: 1440, + accessMode: 'workspace', + credentialGroupId: null, + credentialGroupOptionId: null, + memberSyncStatus: 'idle', + status: 'active', + syncLockToken: null, + memberSyncLockToken: null, +} + +const MEMBERS_CONNECTOR = { + ...WORKSPACE_CONNECTOR, + credentialId: null, + accessMode: 'members', + credentialGroupId: 'group-1', + credentialGroupOptionId: 'option-1', +} + +const BINDING = { + credentialGroupId: 'group-1', + credentialGroupOptionId: 'option-1', + workspaceId: 'ws-1', +} + +function switchTo(target: Parameters[0]['target']) { + return performUpdateKnowledgeConnectorAccess({ + knowledgeBase: KB, + connectorId: 'c-1', + target, + resolveBillingAttribution, + ...ACTOR, + }) +} + +/** The group row the flip locks; `optionIds` are the options it still has. */ +function queueGroupRow(...optionIds: string[]) { + queueTableRows(schemaMock.credentialGroup, [ + { options: optionIds.map((id) => ({ id, provider: 'google-drive', status: 'active' })) }, + ]) +} + +/** The values of the `set()` call that wrote `field`, so a test can read what a later call must repeat. */ +function setCallWith(field: string): Record { + const call = dbChainMockFns.set.mock.calls.find(([values]) => field in values) + if (!call) throw new Error(`No set() call wrote ${field}`) + return call[0] +} + +/** The `set()` calls carrying `field`, in order. */ +function setCallsWith(field: string): Record[] { + return dbChainMockFns.set.mock.calls.filter(([values]) => field in values).map(([v]) => v) +} + +const SCOPED_META = { + name: 'Google Drive', + auth: { mode: 'oauth', provider: 'google-drive' }, + permissionScopedListing: { capFieldIds: [] }, + configFields: [], +} as never + +describe('resolveKnowledgeConnectorMembersBinding', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.memberAccessAvailable.mockResolvedValue(true) + }) + + it('refuses a connector whose listing is not permission-scoped, before loading anything', async () => { + await expect( + resolveKnowledgeConnectorMembersBinding({ + workspaceId: 'ws-1', + connectorMeta: { name: 'Slack', auth: { mode: 'oauth' }, configFields: [] } as never, + actingUserId: 'admin-1', + binding: null, + sourceConfig: {}, + }) + ).rejects.toMatchObject({ code: 'validation' }) + expect(mocks.provision).not.toHaveBeenCalled() + expect(mocks.loadGroup).not.toHaveBeenCalled() + }) + + it('provisions the group when none is named, then validates it like a named one', async () => { + mocks.provision.mockResolvedValue({ + credentialGroupId: 'group-9', + credentialGroupOptionId: 'option-9', + }) + mocks.loadGroup.mockResolvedValue({ workspaceId: 'ws-1', status: 'active', options: [] }) + mocks.validateBinding.mockReturnValue({ ok: true, option: {} }) + await expect( + resolveKnowledgeConnectorMembersBinding({ + workspaceId: 'ws-1', + connectorMeta: SCOPED_META, + actingUserId: 'admin-1', + binding: null, + sourceConfig: {}, + }) + ).resolves.toEqual({ + credentialGroupId: 'group-9', + credentialGroupOptionId: 'option-9', + sourceConfig: {}, + }) + expect(mocks.provision).toHaveBeenCalledWith({ + workspaceId: 'ws-1', + connectorMeta: SCOPED_META, + userId: 'admin-1', + }) + expect(mocks.loadGroup).toHaveBeenCalledWith('group-9') + }) + + it('refuses members mode where the feature is off, before loading anything', async () => { + mocks.memberAccessAvailable.mockResolvedValue(false) + await expect( + resolveKnowledgeConnectorMembersBinding({ + workspaceId: 'ws-1', + connectorMeta: SCOPED_META, + actingUserId: 'admin-1', + binding: { credentialGroupId: 'group-1', credentialGroupOptionId: 'option-1' }, + sourceConfig: {}, + }) + ).rejects.toMatchObject({ message: 'Per-member access is not available for this workspace' }) + expect(mocks.memberAccessAvailable).toHaveBeenCalledWith({ workspaceId: 'ws-1' }) + expect(mocks.loadGroup).not.toHaveBeenCalled() + }) + + it('refuses a group from another workspace before validating anything', async () => { + mocks.loadGroup.mockResolvedValue({ workspaceId: 'ws-2', status: 'active', options: [] }) + await expect( + resolveKnowledgeConnectorMembersBinding({ + workspaceId: 'ws-1', + connectorMeta: SCOPED_META, + actingUserId: 'admin-1', + binding: { credentialGroupId: 'group-1', credentialGroupOptionId: 'option-1' }, + sourceConfig: {}, + }) + ).rejects.toMatchObject({ code: 'validation' }) + expect(mocks.validateBinding).not.toHaveBeenCalled() + }) + + it('surfaces the validator refusal as a validation error', async () => { + mocks.loadGroup.mockResolvedValue({ workspaceId: 'ws-1', status: 'active', options: [] }) + mocks.validateBinding.mockReturnValue({ ok: false, message: 'Max Files cannot be set' }) + await expect( + resolveKnowledgeConnectorMembersBinding({ + workspaceId: 'ws-1', + connectorMeta: SCOPED_META, + actingUserId: 'admin-1', + binding: { credentialGroupId: 'group-1', credentialGroupOptionId: 'option-1' }, + sourceConfig: { maxFiles: '5' }, + }) + ).rejects.toMatchObject({ code: 'validation', message: 'Max Files cannot be set' }) + }) +}) + +describe('performUpdateKnowledgeConnectorAccess', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mocks.rewriteAcls.mockResolvedValue(true) + mocks.grant.mockResolvedValue(undefined) + mocks.revoke.mockResolvedValue(undefined) + mocks.dispatchSync.mockResolvedValue({ queued: true }) + mocks.dispatchMemberSync.mockResolvedValue({ queued: true }) + }) + + it('is a no-op when the connector already has the requested binding', async () => { + queueTableRows(schemaMock.knowledgeConnector, [MEMBERS_CONNECTOR]) + + const outcome = await switchTo({ accessMode: 'members', binding: BINDING }) + + expect(outcome).toMatchObject({ success: true, changed: false }) + expect(dbChainMockFns.update).not.toHaveBeenCalled() + expect(mocks.grant).not.toHaveBeenCalled() + }) + + it('refuses while a sync of either engine owns the connector', async () => { + queueTableRows(schemaMock.knowledgeConnector, [WORKSPACE_CONNECTOR]) + dbChainMockFns.returning.mockResolvedValueOnce([]) + + const outcome = await switchTo({ accessMode: 'members', binding: BINDING }) + + expect(outcome).toEqual({ + success: false, + error: 'Sync already in progress', + errorCode: 'conflict', + }) + expect(mocks.grant).not.toHaveBeenCalled() + }) + + it('hides the documents, grants the option, flips to members mode, and queues the first member run', async () => { + queueTableRows(schemaMock.knowledgeConnector, [WORKSPACE_CONNECTOR]) + queueGroupRow('option-1') + dbChainMockFns.returning + .mockResolvedValueOnce([{ ...WORKSPACE_CONNECTOR, status: 'syncing', syncLockToken: 's-1' }]) + /** The flip lands under the lease, then the release. */ + .mockResolvedValueOnce([{ id: 'c-1' }]) + .mockResolvedValueOnce([{ ...MEMBERS_CONNECTOR, nextMemberSyncAt: new Date() }]) + + const outcome = await switchTo({ accessMode: 'members', binding: BINDING }) + + expect(outcome).toMatchObject({ success: true, changed: true }) + /** The rewrite hides every document and proves the switch lease inside each batch. */ + expect(mocks.rewriteAcls).toHaveBeenCalledWith( + 'c-1', + [], + expect.objectContaining({ + lease: expect.objectContaining({ stillHeld: expect.any(Function) }), + }) + ) + expect(mocks.grant).toHaveBeenCalledWith( + { + workspaceId: 'ws-1', + credentialGroupId: 'group-1', + credentialGroupOptionId: 'option-1', + connectorId: 'c-1', + }, + 'admin-1' + ) + /** The flip is written inside the group's row lock. */ + expect(dbChainMockFns.for).toHaveBeenCalledWith('update') + expect(dbChainMockFns.transaction).toHaveBeenCalledOnce() + const flip = setCallWith('accessMode') + expect(flip).toMatchObject({ + accessMode: 'members', + credentialId: null, + credentialGroupId: 'group-1', + credentialGroupOptionId: 'option-1', + accessRewritePending: false, + nextSyncAt: null, + nextMemberSyncAt: expect.any(Date), + }) + expect(flip).not.toHaveProperty('status') + expect(dbChainMockFns.set).toHaveBeenLastCalledWith( + expect.objectContaining({ status: 'active', syncLockToken: null, syncLockLeaseAt: null }) + ) + /** The dispatch asserts the schedule the flip wrote, so the queue accepts it. */ + expect(mocks.dispatchMemberSync).toHaveBeenCalledWith('c-1', { + billingAttribution: BILLING, + expectedNextMemberSyncAt: flip.nextMemberSyncAt, + requestId: 'req-1', + requireRunnable: true, + }) + expect(mocks.dispatchSync).not.toHaveBeenCalled() + }) + + it('refuses the flip, and undoes the grant, when the option is gone by the time the group is locked', async () => { + queueTableRows(schemaMock.knowledgeConnector, [WORKSPACE_CONNECTOR]) + queueGroupRow() + dbChainMockFns.returning.mockResolvedValueOnce([ + { ...WORKSPACE_CONNECTOR, status: 'syncing', syncLockToken: 's-1' }, + ]) + + const outcome = await switchTo({ accessMode: 'members', binding: BINDING }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'validation' }) + expect(setCallsWith('accessMode')).toEqual([]) + expect(mocks.revoke).toHaveBeenCalledWith( + { workspaceId: 'ws-1', credentialGroupId: 'group-1', connectorId: 'c-1' }, + 'admin-1' + ) + expect(dbChainMockFns.set).toHaveBeenLastCalledWith( + expect.objectContaining({ status: 'active', syncLockToken: null }) + ) + expect(mocks.dispatchMemberSync).not.toHaveBeenCalled() + }) + + it('keeps the lease until the previous group is revoked when moving between groups', async () => { + queueTableRows(schemaMock.knowledgeConnector, [MEMBERS_CONNECTOR]) + queueGroupRow('option-2') + dbChainMockFns.returning + .mockResolvedValueOnce([{ ...MEMBERS_CONNECTOR, status: 'syncing', syncLockToken: 's-1' }]) + .mockResolvedValueOnce([{ id: 'c-1' }]) + .mockResolvedValueOnce([ + { ...MEMBERS_CONNECTOR, credentialGroupId: 'group-2', credentialGroupOptionId: 'option-2' }, + ]) + + const outcome = await switchTo({ + accessMode: 'members', + binding: { + credentialGroupId: 'group-2', + credentialGroupOptionId: 'option-2', + sourceConfig: {}, + }, + }) + + expect(outcome).toMatchObject({ success: true, changed: true }) + expect(mocks.revoke).toHaveBeenCalledWith( + { workspaceId: 'ws-1', credentialGroupId: 'group-1', connectorId: 'c-1' }, + 'admin-1' + ) + /** + * A revoke drops the connector from every option of the group. Released + * first, a switch that had re-granted group-1 in between would lose it. + */ + const revokedAt = mocks.revoke.mock.invocationCallOrder[0] + const releasedAt = dbChainMockFns.set.mock.invocationCallOrder.at(-1) ?? 0 + expect(revokedAt).toBeLessThan(releasedAt) + expect(dbChainMockFns.set).toHaveBeenLastCalledWith( + expect.objectContaining({ status: 'active', syncLockToken: null }) + ) + }) + + it('drops the members, restores workspace access, revokes the grant, flips, and queues a content sync', async () => { + queueTableRows(schemaMock.knowledgeConnector, [MEMBERS_CONNECTOR]) + dbChainMockFns.returning + .mockResolvedValueOnce([{ ...MEMBERS_CONNECTOR, status: 'syncing', syncLockToken: 's-1' }]) + /** The flip lands under the lease, then the release. */ + .mockResolvedValueOnce([{ id: 'c-1' }]) + .mockResolvedValueOnce([{ ...WORKSPACE_CONNECTOR, credentialId: 'cred-2' }]) + + const outcome = await switchTo({ accessMode: 'workspace', credentialId: 'cred-2' }) + + expect(outcome).toMatchObject({ success: true, changed: true }) + expect(dbChainMockFns.delete).toHaveBeenCalled() + /** Workspace access is restored under the switch lease, proved inside each batch. */ + expect(mocks.rewriteAcls).toHaveBeenCalledWith( + 'c-1', + ['ws'], + expect.objectContaining({ + lease: expect.objectContaining({ stillHeld: expect.any(Function) }), + }) + ) + expect(mocks.revoke).toHaveBeenCalledWith( + { workspaceId: 'ws-1', credentialGroupId: 'group-1', connectorId: 'c-1' }, + 'admin-1' + ) + const flip = setCallWith('accessMode') + expect(flip).toMatchObject({ + accessMode: 'workspace', + credentialId: 'cred-2', + credentialGroupId: null, + credentialGroupOptionId: null, + nextMemberSyncAt: null, + nextSyncAt: expect.any(Date), + }) + /** The revoke lands while the lease is still held; the release is the last write. */ + const revokedAt = mocks.revoke.mock.invocationCallOrder[0] + const releasedAt = dbChainMockFns.set.mock.invocationCallOrder.at(-1) ?? 0 + expect(revokedAt).toBeLessThan(releasedAt) + expect(dbChainMockFns.set).toHaveBeenLastCalledWith( + expect.objectContaining({ + accessRewritePending: false, + status: 'active', + syncLockToken: null, + syncLockLeaseAt: null, + }) + ) + /** + * The row holds the instant the flip wrote as `nextSyncAt`; a dispatch + * asserting any later clock read is refused by the queue as stale. + */ + expect(mocks.dispatchSync).toHaveBeenCalledWith('c-1', { + billingAttribution: BILLING, + expectedNextSyncAt: flip.nextSyncAt, + requestId: 'req-1', + requireRunnable: true, + }) + expect(mocks.dispatchMemberSync).not.toHaveBeenCalled() + }) + + it('changes a workspace credential without the lease, drops the watermark, and queues a full sync', async () => { + queueTableRows(schemaMock.knowledgeConnector, [ + { ...WORKSPACE_CONNECTOR, lastSyncAt: new Date('2026-08-01T00:00:00Z') }, + ]) + dbChainMockFns.returning.mockResolvedValueOnce([ + { ...WORKSPACE_CONNECTOR, credentialId: 'cred-2', lastSyncAt: null }, + ]) + + const outcome = await switchTo({ accessMode: 'workspace', credentialId: 'cred-2' }) + + expect(outcome).toMatchObject({ success: true, changed: true }) + expect(dbChainMockFns.update).toHaveBeenCalledOnce() + const change = setCallWith('credentialId') + expect(change).toMatchObject({ + credentialId: 'cred-2', + lastSyncAt: null, + nextSyncAt: expect.any(Date), + }) + expect(change).not.toHaveProperty('status') + /** + * A running sync's terminal write would restore the watermark, so the + * write is refused while any sync holds the row. + */ + expect( + hasMockCondition( + dbChainMockFns.where.mock.calls.at(-1)?.[0], + (node: MockCondition) => + node.type === 'isNull' && node.column === schemaMock.knowledgeConnector.syncLockToken + ) + ).toBe(true) + expect(mocks.dispatchSync).toHaveBeenCalledWith('c-1', { + billingAttribution: BILLING, + expectedNextSyncAt: change.nextSyncAt, + requestId: 'req-1', + requireRunnable: true, + }) + expect(mocks.grant).not.toHaveBeenCalled() + expect(mocks.revoke).not.toHaveBeenCalled() + }) + + it('refuses a credential change while a sync owns the connector', async () => { + queueTableRows(schemaMock.knowledgeConnector, [WORKSPACE_CONNECTOR]) + queueTableRows(schemaMock.knowledgeConnector, [ + { ...WORKSPACE_CONNECTOR, status: 'syncing', syncLockToken: 'run-1' }, + ]) + dbChainMockFns.returning.mockResolvedValueOnce([]) + + const outcome = await switchTo({ accessMode: 'workspace', credentialId: 'cred-2' }) + + expect(outcome).toEqual({ + success: false, + error: 'Sync already in progress', + errorCode: 'conflict', + }) + expect(mocks.dispatchSync).not.toHaveBeenCalled() + }) + + it('changes the credential of a paused connector without queuing a sync', async () => { + queueTableRows(schemaMock.knowledgeConnector, [{ ...WORKSPACE_CONNECTOR, status: 'paused' }]) + dbChainMockFns.returning.mockResolvedValueOnce([ + { ...WORKSPACE_CONNECTOR, status: 'paused', credentialId: 'cred-2' }, + ]) + + const outcome = await switchTo({ accessMode: 'workspace', credentialId: 'cred-2' }) + + expect(outcome).toMatchObject({ success: true, changed: true }) + expect(dbChainMockFns.set).toHaveBeenCalledWith( + expect.objectContaining({ credentialId: 'cred-2', lastSyncAt: null }) + ) + expect(mocks.dispatchSync).not.toHaveBeenCalled() + }) + + it('leaves a paused connector paused and queues nothing', async () => { + queueTableRows(schemaMock.knowledgeConnector, [{ ...WORKSPACE_CONNECTOR, status: 'paused' }]) + queueGroupRow('option-1') + dbChainMockFns.returning + .mockResolvedValueOnce([{ ...WORKSPACE_CONNECTOR, status: 'paused' }]) + .mockResolvedValueOnce([{ id: 'c-1' }]) + .mockResolvedValueOnce([{ ...MEMBERS_CONNECTOR, status: 'paused' }]) + + const outcome = await switchTo({ accessMode: 'members', binding: BINDING }) + + expect(outcome).toMatchObject({ success: true, changed: true }) + expect(dbChainMockFns.set).toHaveBeenCalledWith( + expect.objectContaining({ accessMode: 'members' }) + ) + expect(dbChainMockFns.set).toHaveBeenLastCalledWith( + expect.objectContaining({ status: 'paused', syncLockToken: null }) + ) + expect(mocks.dispatchMemberSync).not.toHaveBeenCalled() + }) + + it('releases the lease and reports the failure when the grant cannot be written', async () => { + queueTableRows(schemaMock.knowledgeConnector, [WORKSPACE_CONNECTOR]) + dbChainMockFns.returning + .mockResolvedValueOnce([{ ...WORKSPACE_CONNECTOR, status: 'syncing', syncLockToken: 's-1' }]) + .mockResolvedValueOnce([]) + mocks.grant.mockRejectedValueOnce(new Error('policy store unavailable')) + + const outcome = await switchTo({ accessMode: 'members', binding: BINDING }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'internal' }) + expect(dbChainMockFns.set).toHaveBeenLastCalledWith( + expect.objectContaining({ status: 'active', syncLockToken: null, syncLockLeaseAt: null }) + ) + expect(dbChainMockFns.set).not.toHaveBeenCalledWith( + expect.objectContaining({ accessMode: 'members' }) + ) + }) +}) diff --git a/apps/sim/lib/knowledge/orchestration/connector-access.ts b/apps/sim/lib/knowledge/orchestration/connector-access.ts new file mode 100644 index 00000000000..5cfbd6773da --- /dev/null +++ b/apps/sim/lib/knowledge/orchestration/connector-access.ts @@ -0,0 +1,573 @@ +import { db } from '@sim/db' +import { knowledgeConnector, knowledgeConnectorMember } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { and, eq, inArray, isNull } from 'drizzle-orm' +import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attribution' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { generateRequestId } from '@/lib/core/utils/request' +import { loadCredentialGroupCredentialListContext } from '@/lib/credential-groups/credentials' +import { requireKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' +import { EMPTY_ACL, WORKSPACE_ACL } from '@/lib/knowledge/access/tokens' +import { + grantKnowledgeConnectorCredentialAccess, + revokeKnowledgeConnectorCredentialAccess, + stripListingCapFields, + validateKnowledgeConnectorMembersBinding, +} from '@/lib/knowledge/connectors/member-access' +import { rewriteConnectorAcls } from '@/lib/knowledge/connectors/member-observations' +import { provisionKnowledgeConnectorMembersBinding } from '@/lib/knowledge/connectors/member-provisioning' +import { + type ConnectorWithoutSecret, + getKnowledgeConnector, + type KnowledgeConnectorRow, + lockCredentialGroupOption, +} from '@/lib/knowledge/orchestration/connectors' +import { + classifyKnowledgeFailure, + fail, + type KnowledgeOperationContext, + type KnowledgeOrchestrationResult, +} from '@/lib/knowledge/orchestration/shared' +import type { ConnectorMeta } from '@/connectors/types' + +const logger = createLogger('KnowledgeConnectorAccessOrchestration') + +/** The switch lease was taken away between acquiring it and writing the flip. */ +class SwitchLeaseLostError extends Error { + constructor() { + super('Connector changed during the switch') + this.name = 'SwitchLeaseLostError' + } +} + +/** Wall-clock the request spends rewriting before handing the rest to the member run. */ +const ACCESS_REWRITE_REQUEST_BUDGET_MS = 20_000 +/** Connector statuses a switch may start from; a running or queued sync owns the row. */ +const SWITCHABLE_CONNECTOR_STATUSES = ['active', 'error', 'paused'] as const + +async function loadDispatchSync() { + return (await import('@/lib/knowledge/connectors/queue')).dispatchSync +} + +async function loadDispatchMemberSync() { + return (await import('@/lib/knowledge/connectors/member-queue')).dispatchMemberSync +} + +/** The credential-group binding a members-mode connector needs, as the caller supplied it. */ +export interface KnowledgeConnectorMembersBinding { + credentialGroupId: string + credentialGroupOptionId: string +} + +export interface ResolvedMembersBinding extends KnowledgeConnectorMembersBinding { + /** The connector's source config with the listing caps cleared, which members mode stores. */ + sourceConfig: Record +} + +/** + * Checks a members-mode binding against the group, the option, and the + * connector, before any row is touched. Shared by creation and by the mode + * switch, so both refuse exactly the same bindings. + */ +export async function resolveKnowledgeConnectorMembersBinding(input: { + workspaceId: string + connectorMeta: Pick + /** The option the caller named, or null to sync through the workspace's group for the provider, created if need be. */ + binding: KnowledgeConnectorMembersBinding | null + /** The admin acting, recorded as the creator of a provisioned group. */ + actingUserId: string + sourceConfig: Record +}): Promise { + /** + * Judged by the workspace alone, as the member engine is: a person's own + * flag clause must not open a mode the engine will then refuse to run. + */ + await requireKnowledgeMemberAccessAvailable({ workspaceId: input.workspaceId }) + if (!input.connectorMeta.permissionScopedListing) { + throw new OrchestrationError( + 'validation', + `${input.connectorMeta.name} cannot sync per member: its listing does not reflect who may read each document` + ) + } + const sourceConfig = stripListingCapFields(input.connectorMeta, input.sourceConfig) + const binding = + input.binding ?? + (await provisionKnowledgeConnectorMembersBinding({ + workspaceId: input.workspaceId, + connectorMeta: input.connectorMeta, + userId: input.actingUserId, + })) + const group = await loadCredentialGroupCredentialListContext(binding.credentialGroupId) + if (!group || group.workspaceId !== input.workspaceId) { + throw new OrchestrationError('validation', 'Credential Group was not found in this workspace') + } + const validation = validateKnowledgeConnectorMembersBinding({ + connectorMeta: input.connectorMeta, + group, + credentialGroupOptionId: binding.credentialGroupOptionId, + sourceConfig, + }) + if (!validation.ok) throw new OrchestrationError('validation', validation.message) + return { ...binding, sourceConfig } +} + +/** + * Takes the connector's content lease for the switch, so no sync of either + * engine can start while documents are being rewritten. Returns the row as it + * was, or null when a sync already owns it. + */ +async function acquireSwitchLease( + connectorId: string, + knowledgeBaseId: string, + switchId: string, + expectedStatus: string +): Promise { + const now = new Date() + const [row] = await db + .update(knowledgeConnector) + .set({ status: 'syncing', syncLockToken: switchId, syncLockLeaseAt: now, updatedAt: now }) + .where( + and( + eq(knowledgeConnector.id, connectorId), + eq(knowledgeConnector.knowledgeBaseId, knowledgeBaseId), + inArray(knowledgeConnector.status, SWITCHABLE_CONNECTOR_STATUSES), + eq(knowledgeConnector.status, expectedStatus), + inArray(knowledgeConnector.memberSyncStatus, ['idle', 'error', 'disabled']), + isNull(knowledgeConnector.syncLockToken), + isNull(knowledgeConnector.memberSyncLockToken), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + .returning() + return row ?? null +} + +function switchLeaseHeld(connectorId: string, switchId: string) { + return and( + eq(knowledgeConnector.id, connectorId), + eq(knowledgeConnector.status, 'syncing'), + eq(knowledgeConnector.syncLockToken, switchId) + ) +} + +/** + * Hands the lease back, restoring the status the switch found, along with any + * last values the switch writes as it ends. Returns the row as released, or + * null when the lease had already been taken away. + */ +async function releaseSwitchLease( + connectorId: string, + switchId: string, + previousStatus: string, + values: Partial = {} +): Promise { + const [row] = await db + .update(knowledgeConnector) + .set({ + ...values, + status: previousStatus, + syncLockToken: null, + syncLockLeaseAt: null, + updatedAt: new Date(), + }) + .where(switchLeaseHeld(connectorId, switchId)) + .returning() + return row ?? null +} + +export interface PerformUpdateKnowledgeConnectorAccessParams extends KnowledgeOperationContext { + knowledgeBase: { id: string; name: string; workspaceId: string } + connectorId: string + target: + | { accessMode: 'members'; binding: ResolvedMembersBinding } + | { accessMode: 'workspace'; credentialId: string } + resolveBillingAttribution: () => Promise +} + +export type PerformUpdateKnowledgeConnectorAccessResult = KnowledgeOrchestrationResult<{ + connector: ConnectorWithoutSecret + /** Whether the switch changed anything; a repeat of the current binding is a no-op. */ + changed: boolean +}> + +/** + * Moves a connector between access modes under the connector's content lease, + * so neither engine runs against a half-rewritten corpus. + * + * Into members mode: grant the option's credentials first (a reversible policy + * write), rewrite every ACL to nobody, flip under the Credential Group's row + * lock, then revoke the previous group's grant and release. A rewrite that + * outgrows the request budget is finished by the first member run before it + * lists (`accessRewritePending`); documents are hidden early, never shown + * early. + * + * Back to workspace mode: drop the members and flip in one transaction with + * the rewrite marked pending, rewrite every ACL to the workspace while the + * lease is still held, then revoke the grant and release. A rewrite that + * outgrows the budget, or is interrupted, is finished by the next content + * sync (`accessRewritePending`); documents are hidden until then. + * + * Either way the lease outlives the revoke. A revoke drops the connector from + * every option of the group, so releasing first would let a switch that has + * just re-granted the same group lose its grant to this one's cleanup. + */ +export async function performUpdateKnowledgeConnectorAccess( + params: PerformUpdateKnowledgeConnectorAccessParams +): Promise { + const { knowledgeBase: kb, connectorId, target } = params + const requestId = params.requestId ?? generateRequestId() + + const existing = await getKnowledgeConnector(kb.id, connectorId) + if (!existing) return fail('Connector not found', 'not_found') + + const unchanged = + target.accessMode === existing.accessMode && + (target.accessMode === 'workspace' + ? target.credentialId === existing.credentialId + : target.binding.credentialGroupId === existing.credentialGroupId && + target.binding.credentialGroupOptionId === existing.credentialGroupOptionId) + if (unchanged) { + /** + * Re-applying the current binding on a connector whose member sync was + * disabled is how it is re-enabled: the next run reconciles members from + * the group again and restores access from the retained observations. + */ + if (target.accessMode === 'members' && existing.memberSyncStatus === 'disabled') { + const now = new Date() + const [updated] = await db + .update(knowledgeConnector) + .set({ + memberSyncStatus: 'idle', + memberSyncConsecutiveFailures: 0, + lastMemberSyncError: null, + nextMemberSyncAt: now, + updatedAt: now, + }) + .where( + and( + eq(knowledgeConnector.id, connectorId), + eq(knowledgeConnector.memberSyncStatus, 'disabled'), + isNull(knowledgeConnector.deletedAt) + ) + ) + .returning() + if (!updated) return fail('Connector changed; retry the request', 'conflict') + logger.info(`[${requestId}] Re-enabled member sync on connector ${connectorId}`) + const { encryptedApiKey: _secret, ...connector } = updated + if (updated.status !== 'paused') { + await dispatchMemberSyncBestEffort(connectorId, params, requestId, now) + } + return { success: true, connector, changed: true } + } + const { encryptedApiKey: _secret, ...connector } = existing + return { success: true, connector, changed: false } + } + + /** + * Staying in workspace mode with a different credential moves no document's + * visibility, so the lease is not taken. It does change what the source + * shows: the new credential may see a different corpus, and only a full + * listing reconciles that, so the incremental watermark is dropped and a sync + * queued. The write refuses while a sync owns the row, whose terminal write + * would otherwise put the watermark straight back. + */ + if (target.accessMode === 'workspace' && existing.accessMode === 'workspace') { + const now = new Date() + const [updated] = await db + .update(knowledgeConnector) + .set({ + credentialId: target.credentialId, + lastSyncAt: null, + nextSyncAt: now, + updatedAt: now, + }) + .where( + and( + eq(knowledgeConnector.id, connectorId), + eq(knowledgeConnector.knowledgeBaseId, kb.id), + inArray(knowledgeConnector.status, SWITCHABLE_CONNECTOR_STATUSES), + eq(knowledgeConnector.status, existing.status), + isNull(knowledgeConnector.syncLockToken), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + .returning() + if (!updated) { + const current = await getKnowledgeConnector(kb.id, connectorId) + return current + ? fail('Sync already in progress', 'conflict') + : fail('Connector not found', 'not_found') + } + logger.info(`[${requestId}] Changed the credential of connector ${connectorId}`) + const { encryptedApiKey: _secret, ...connector } = updated + if (existing.status !== 'paused') { + await dispatchContentSyncBestEffort(connectorId, params, requestId, now) + } + return { success: true, connector, changed: true } + } + + const switchId = generateId() + /** + * The status to restore is the one the row had before the lease, which the + * lease itself asserts: a status that moved between the read and the lease + * makes the lease fail rather than be restored wrongly. + */ + const previousStatus = existing.status + const leased = await acquireSwitchLease(connectorId, kb.id, switchId, existing.status) + if (!leased) return fail('Sync already in progress', 'conflict') + const deadlineAt = Date.now() + ACCESS_REWRITE_REQUEST_BUDGET_MS + + try { + if (target.accessMode === 'members') { + await grantKnowledgeConnectorCredentialAccess( + { + workspaceId: kb.workspaceId, + credentialGroupId: target.binding.credentialGroupId, + credentialGroupOptionId: target.binding.credentialGroupOptionId, + connectorId, + }, + params.userId + ) + try { + const rewritten = await rewriteConnectorAcls(connectorId, EMPTY_ACL, { + deadlineAt: deadlineAt, + lease: { stillHeld: () => switchLeaseHeld(connectorId, switchId) }, + }) + /** + * The flip lands under the group's row lock, which the group's option + * edits and delete hold while they look for connectors bound to what + * they remove: an option gone by the time the lock is ours refuses the + * flip, and one removed after it finds this row. + */ + const flippedAt = new Date() + await db.transaction(async (tx) => { + await lockCredentialGroupOption(tx, { + workspaceId: kb.workspaceId, + credentialGroupId: target.binding.credentialGroupId, + credentialGroupOptionId: target.binding.credentialGroupOptionId, + }) + const [row] = await tx + .update(knowledgeConnector) + .set({ + accessMode: 'members', + credentialId: null, + credentialGroupId: target.binding.credentialGroupId, + credentialGroupOptionId: target.binding.credentialGroupOptionId, + sourceConfig: target.binding.sourceConfig, + accessRewritePending: !rewritten, + memberSyncStatus: 'idle', + memberSyncConsecutiveFailures: 0, + lastMemberSyncError: null, + nextMemberSyncAt: flippedAt, + nextSyncAt: null, + updatedAt: flippedAt, + }) + .where(switchLeaseHeld(connectorId, switchId)) + .returning({ id: knowledgeConnector.id }) + if (!row) throw new SwitchLeaseLostError() + }) + if ( + existing.credentialGroupId && + existing.credentialGroupId !== target.binding.credentialGroupId + ) { + await revokeKnowledgeConnectorCredentialAccess( + { + workspaceId: kb.workspaceId, + credentialGroupId: existing.credentialGroupId, + connectorId, + }, + params.userId + ).catch((error) => { + logger.error(`[${requestId}] Failed to revoke the previous group's grant`, { + connectorId, + error: getErrorMessage(error), + }) + }) + } + const updated = await releaseSwitchLease(connectorId, switchId, previousStatus) + if (!updated) throw new SwitchLeaseLostError() + logger.info(`[${requestId}] Switched connector ${connectorId} to members mode`, { + rewritten, + }) + const { encryptedApiKey: _secret, ...connector } = updated + if (previousStatus !== 'paused') { + await dispatchMemberSyncBestEffort(connectorId, params, requestId, flippedAt) + } + return { success: true, connector, changed: true } + } catch (error) { + /** + * The grant is the one write a failed switch must not leave behind. A + * grant replaces the connector's option within the group, so a failed + * move between options of one group puts the previous option back + * rather than leaving the connector on none. + */ + const previousOptionId = + existing.credentialGroupId === target.binding.credentialGroupId + ? existing.credentialGroupOptionId + : null + await (previousOptionId + ? grantKnowledgeConnectorCredentialAccess( + { + workspaceId: kb.workspaceId, + credentialGroupId: target.binding.credentialGroupId, + credentialGroupOptionId: previousOptionId, + connectorId, + }, + params.userId + ) + : revokeKnowledgeConnectorCredentialAccess( + { + workspaceId: kb.workspaceId, + credentialGroupId: target.binding.credentialGroupId, + connectorId, + }, + params.userId + ) + ).catch((undoError) => { + logger.error(`[${requestId}] Failed to undo the grant of an abandoned switch`, { + connectorId, + error: getErrorMessage(undoError), + }) + }) + throw error + } + } + + /** + * The flip lands first, still under the lease and with the rewrite marked + * pending, so an interruption anywhere after it leaves a workspace-mode + * connector whose next content sync finishes the rewrite; documents are + * hidden until then, never shown under the wrong mode. + */ + const flippedAt = new Date() + await db.transaction(async (tx) => { + await tx + .delete(knowledgeConnectorMember) + .where(eq(knowledgeConnectorMember.connectorId, connectorId)) + const [row] = await tx + .update(knowledgeConnector) + .set({ + accessMode: 'workspace', + credentialId: target.credentialId, + credentialGroupId: null, + credentialGroupOptionId: null, + accessRewritePending: true, + /** + * The next content sync must list everything and reconcile: the + * union of every member's documents may hold documents the + * workspace credential cannot see, and only a full listing removes + * them. + */ + lastSyncAt: null, + memberSyncStatus: 'idle', + memberSyncConsecutiveFailures: 0, + lastMemberSyncError: null, + nextMemberSyncAt: null, + nextSyncAt: flippedAt, + updatedAt: flippedAt, + }) + .where(switchLeaseHeld(connectorId, switchId)) + .returning({ id: knowledgeConnector.id }) + if (!row) throw new SwitchLeaseLostError() + }) + const rewritten = await rewriteConnectorAcls(connectorId, WORKSPACE_ACL, { + deadlineAt: deadlineAt, + lease: { stillHeld: () => switchLeaseHeld(connectorId, switchId) }, + }) + if (existing.credentialGroupId) { + await revokeKnowledgeConnectorCredentialAccess( + { workspaceId: kb.workspaceId, credentialGroupId: existing.credentialGroupId, connectorId }, + params.userId + ).catch((error) => { + logger.error(`[${requestId}] Failed to revoke the grant after leaving members mode`, { + connectorId, + error: getErrorMessage(error), + }) + }) + } + const updated = await releaseSwitchLease(connectorId, switchId, previousStatus, { + accessRewritePending: !rewritten, + }) + if (!updated) throw new SwitchLeaseLostError() + logger.info(`[${requestId}] Switched connector ${connectorId} to workspace mode`, { + rewritten, + }) + const { encryptedApiKey: _secret, ...connector } = updated + if (previousStatus !== 'paused') { + /** The dispatch asserts the schedule the flip wrote, not a later clock read. */ + await dispatchContentSyncBestEffort(connectorId, params, requestId, flippedAt) + } + return { success: true, connector, changed: true } + } catch (error) { + if (error instanceof SwitchLeaseLostError) { + /** The flip may already have landed; a reaped lease means the content engine now owns the rest. */ + return fail('Connector changed during the switch; retry the request', 'conflict') + } + await releaseSwitchLease(connectorId, switchId, previousStatus).catch((releaseError) => { + logger.error(`[${requestId}] Failed to release the access switch lease`, { + connectorId, + error: releaseError, + }) + return null + }) + return classifyKnowledgeFailure( + error, + requestId, + `Switch access mode of connector ${connectorId}` + ) + } +} + +async function dispatchMemberSyncBestEffort( + connectorId: string, + params: PerformUpdateKnowledgeConnectorAccessParams, + requestId: string, + expectedNextMemberSyncAt: Date +): Promise { + try { + const dispatchMemberSync = await loadDispatchMemberSync() + const dispatch = await dispatchMemberSync(connectorId, { + billingAttribution: await params.resolveBillingAttribution(), + expectedNextMemberSyncAt, + requestId, + requireRunnable: true, + }) + if (!dispatch.queued) { + logger.warn(`[${requestId}] Member sync after the switch was not queued: ${dispatch.reason}`) + } + } catch (error) { + logger.error(`[${requestId}] Failed to dispatch the member sync after the switch`, { + connectorId, + error, + }) + } +} + +async function dispatchContentSyncBestEffort( + connectorId: string, + params: PerformUpdateKnowledgeConnectorAccessParams, + requestId: string, + expectedNextSyncAt: Date +): Promise { + try { + const dispatchSync = await loadDispatchSync() + const dispatch = await dispatchSync(connectorId, { + billingAttribution: await params.resolveBillingAttribution(), + expectedNextSyncAt, + requestId, + requireRunnable: true, + }) + if (!dispatch.queued) { + logger.warn(`[${requestId}] Sync after the switch was not queued: ${dispatch.reason}`) + } + } catch (error) { + logger.error(`[${requestId}] Failed to dispatch the sync after the switch`, { + connectorId, + error, + }) + } +} diff --git a/apps/sim/lib/knowledge/orchestration/connectors.test.ts b/apps/sim/lib/knowledge/orchestration/connectors.test.ts index 3d2732e3fcc..fb0bb804e28 100644 --- a/apps/sim/lib/knowledge/orchestration/connectors.test.ts +++ b/apps/sim/lib/knowledge/orchestration/connectors.test.ts @@ -15,11 +15,17 @@ import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { mockCaptureServerEvent, mockDispatchSync, + mockDispatchMemberSync, + mockGrant, + mockRevoke, mockHasWorkspaceLiveSyncAccess, mockRecordAudit, } = vi.hoisted(() => ({ mockCaptureServerEvent: vi.fn(), mockDispatchSync: vi.fn(), + mockDispatchMemberSync: vi.fn(), + mockGrant: vi.fn(), + mockRevoke: vi.fn(), mockHasWorkspaceLiveSyncAccess: vi.fn(), mockRecordAudit: vi.fn(), })) @@ -39,6 +45,14 @@ vi.mock('@/lib/billing/core/subscription', () => ({ hasWorkspaceLiveSyncAccess: mockHasWorkspaceLiveSyncAccess, })) vi.mock('@/lib/knowledge/connectors/queue', () => ({ dispatchSync: mockDispatchSync })) +vi.mock('@/lib/knowledge/connectors/member-queue', () => ({ + dispatchMemberSync: mockDispatchMemberSync, +})) +vi.mock('@/lib/knowledge/connectors/member-access', () => ({ + grantKnowledgeConnectorCredentialAccess: mockGrant, + revokeKnowledgeConnectorCredentialAccess: mockRevoke, + findListingCapViolation: vi.fn(() => null), +})) vi.mock('@/lib/knowledge/documents/service', () => ({ deleteDocumentStorageFiles: vi.fn().mockResolvedValue(undefined), })) @@ -53,6 +67,12 @@ vi.mock('@/connectors/registry.server', () => ({ auth: { mode: 'apiKey', optional: true }, validateConfig: vi.fn().mockResolvedValue({ valid: true }), }, + google_drive: { + name: 'Google Drive', + auth: { mode: 'oauth', provider: 'google-drive' }, + permissionScopedListing: { capFieldIds: [] }, + validateConfig: vi.fn().mockResolvedValue({ valid: true }), + }, }, })) @@ -919,3 +939,327 @@ describe('performSyncKnowledgeConnector', () => { expect(mockCaptureServerEvent).not.toHaveBeenCalled() }) }) + +describe('members-mode connectors', () => { + const MEMBERS_CONNECTOR = { + id: 'c-1', + knowledgeBaseId: 'kb-1', + connectorType: 'notion', + credentialId: null, + encryptedApiKey: null, + sourceConfig: {}, + syncMode: 'full', + syncIntervalMinutes: 1440, + status: 'active', + accessMode: 'members', + credentialGroupId: 'group-1', + credentialGroupOptionId: 'option-1', + memberSyncStatus: 'idle', + lastMemberSyncError: null, + } + + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + mockDispatchSync.mockResolvedValue({ queued: true }) + mockDispatchMemberSync.mockResolvedValue({ queued: true }) + mockGrant.mockResolvedValue(undefined) + mockRevoke.mockResolvedValue(undefined) + }) + + it('refuses to keep the documents of a connector that syncs per member', async () => { + queueTableRows(schemaMock.knowledgeConnector, [MEMBERS_CONNECTOR]) + + const outcome = await performDeleteKnowledgeConnector({ + knowledgeBase: KB, + connectorId: 'c-1', + deleteDocuments: false, + ...ACTOR, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'conflict' }) + expect(dbChainMockFns.delete).not.toHaveBeenCalled() + }) + + it('revokes the credential grant once the connector and its documents are gone', async () => { + queueTableRows(schemaMock.knowledgeConnector, [MEMBERS_CONNECTOR]) + queueTableRows(schemaMock.document, []) + dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'c-1' }]) + + const outcome = await performDeleteKnowledgeConnector({ + knowledgeBase: KB, + connectorId: 'c-1', + deleteDocuments: true, + ...ACTOR, + }) + + expect(outcome).toMatchObject({ success: true }) + expect(mockRevoke).toHaveBeenCalledWith( + { workspaceId: 'ws-1', credentialGroupId: 'group-1', connectorId: 'c-1' }, + 'user-1' + ) + }) + + it('routes a manual sync of a members-mode connector to the member queue', async () => { + queueTableRows(schemaMock.knowledgeConnector, [MEMBERS_CONNECTOR]) + + const outcome = await performSyncKnowledgeConnector({ + knowledgeBase: KB, + connectorId: 'c-1', + resolveBillingAttribution, + ...ACTOR, + }) + + expect(outcome).toEqual({ success: true }) + expect(mockDispatchMemberSync).toHaveBeenCalledWith('c-1', { + billingAttribution: BILLING, + requestId: 'req-1', + }) + expect(mockDispatchSync).not.toHaveBeenCalled() + }) + + it('refuses a manual sync while a member run is queued or running', async () => { + queueTableRows(schemaMock.knowledgeConnector, [ + { ...MEMBERS_CONNECTOR, memberSyncStatus: 'running' }, + ]) + + const outcome = await performSyncKnowledgeConnector({ + knowledgeBase: KB, + connectorId: 'c-1', + resolveBillingAttribution, + ...ACTOR, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'conflict' }) + expect(mockDispatchMemberSync).not.toHaveBeenCalled() + }) + + /** A CAS clause of the last connector update, found by shape since the mock evaluates nothing. */ + function updateCasHas(predicate: (node: MockCondition) => boolean): boolean { + return hasMockCondition(dbChainMockFns.where.mock.calls.at(-1)?.[0], predicate) + } + + it('writes an interval change to the member schedule, which is what the member scheduler reads', async () => { + queueTableRows(schemaMock.knowledgeConnector, [ + { ...MEMBERS_CONNECTOR, nextSyncAt: null, nextMemberSyncAt: null }, + ]) + dbChainMockFns.returning.mockResolvedValueOnce([MEMBERS_CONNECTOR]) + + const outcome = await performUpdateKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'c-1', + updates: { syncIntervalMinutes: 60 }, + resolveBillingAttribution, + }) + + expect(outcome).toMatchObject({ success: true }) + const values = dbChainMockFns.set.mock.calls[0][0] + expect(values).toMatchObject({ syncIntervalMinutes: 60, nextMemberSyncAt: expect.any(Date) }) + expect(values).not.toHaveProperty('nextSyncAt') + expect( + updateCasHas( + (node) => + node.type === 'isNull' && node.column === schemaMock.knowledgeConnector.nextMemberSyncAt + ) + ).toBe(true) + expect(mockDispatchMemberSync).not.toHaveBeenCalled() + }) + + it('clears the member schedule when scheduled sync is turned off', async () => { + const scheduled = new Date(Date.now() + 60 * 60 * 1000) + queueTableRows(schemaMock.knowledgeConnector, [ + { ...MEMBERS_CONNECTOR, nextSyncAt: null, nextMemberSyncAt: scheduled }, + ]) + dbChainMockFns.returning.mockResolvedValueOnce([MEMBERS_CONNECTOR]) + + await performUpdateKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'c-1', + updates: { syncIntervalMinutes: 0 }, + resolveBillingAttribution, + }) + + expect(dbChainMockFns.set.mock.calls[0][0]).toMatchObject({ + syncIntervalMinutes: 0, + nextMemberSyncAt: null, + }) + expect( + updateCasHas( + (node) => + node.type === 'eq' && + node.left === schemaMock.knowledgeConnector.nextMemberSyncAt && + node.right === scheduled + ) + ).toBe(true) + }) + + it('resumes a paused members-mode connector by making its member run due', async () => { + queueTableRows(schemaMock.knowledgeConnector, [ + { ...MEMBERS_CONNECTOR, status: 'paused', nextSyncAt: null, nextMemberSyncAt: null }, + ]) + dbChainMockFns.returning.mockResolvedValueOnce([MEMBERS_CONNECTOR]) + + const outcome = await performUpdateKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'c-1', + updates: { status: 'active' }, + resolveBillingAttribution, + }) + + expect(outcome).toMatchObject({ success: true }) + const values = dbChainMockFns.set.mock.calls[0][0] + expect(values).toMatchObject({ status: 'active', nextMemberSyncAt: expect.any(Date) }) + expect(values).not.toHaveProperty('nextSyncAt') + }) + + it('refuses any edit while a member run is running', async () => { + queueTableRows(schemaMock.knowledgeConnector, [ + { ...MEMBERS_CONNECTOR, memberSyncStatus: 'running' }, + ]) + + const outcome = await performUpdateKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'c-1', + updates: { sourceConfig: { database: 'other' } }, + resolveBillingAttribution, + }) + + /** + * The member run reads `sourceConfig` once at its start and reconciles + * against it, exactly as the content engine does; `status` stays `active` + * throughout, so only the member lease shows the row is owned. + */ + expect(outcome).toMatchObject({ + success: false, + errorCode: 'conflict', + error: 'Sync already in progress', + }) + expect(dbChainMockFns.update).not.toHaveBeenCalled() + }) + + it('refuses a config edit while a member run is queued, but lets a pause release the entry', async () => { + queueTableRows(schemaMock.knowledgeConnector, [ + { ...MEMBERS_CONNECTOR, memberSyncStatus: 'pending', memberSyncLockToken: 'd-1' }, + ]) + + const refused = await performUpdateKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'c-1', + updates: { syncIntervalMinutes: 30 }, + resolveBillingAttribution, + }) + expect(refused).toMatchObject({ success: false, errorCode: 'conflict' }) + expect(dbChainMockFns.update).not.toHaveBeenCalled() + + queueTableRows(schemaMock.knowledgeConnector, [ + { ...MEMBERS_CONNECTOR, memberSyncStatus: 'pending', memberSyncLockToken: 'd-1' }, + ]) + dbChainMockFns.returning.mockResolvedValueOnce([{ ...MEMBERS_CONNECTOR, status: 'paused' }]) + + const paused = await performUpdateKnowledgeConnector({ + ...ACTOR, + knowledgeBase: KB, + connectorId: 'c-1', + updates: { status: 'paused' }, + resolveBillingAttribution, + }) + + /** + * The queued task starts without re-checking `status`, so the entry has to + * go for the pause to hold; the CAS keeps that off a run that has started. + */ + expect(paused).toMatchObject({ success: true }) + expect(dbChainMockFns.set).toHaveBeenCalledWith( + expect.objectContaining({ + status: 'paused', + memberSyncStatus: 'idle', + memberSyncLockToken: null, + memberSyncLockLeaseAt: null, + }) + ) + expect( + updateCasHas( + (node) => + node.type === 'eq' && + node.left === schemaMock.knowledgeConnector.memberSyncStatus && + node.right === 'pending' + ) + ).toBe(true) + }) + + it('binds a new members-mode connector under the group lock', async () => { + queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb-1' }]) + queueTableRows(schemaMock.credentialGroup, [{ options: [{ id: 'option-1' }] }]) + dbChainMockFns.returning.mockResolvedValueOnce([ + { ...MEMBERS_CONNECTOR, connectorType: 'google_drive' }, + ]) + + const outcome = await performCreateKnowledgeConnector({ + knowledgeBase: KB, + connectorType: 'google_drive', + sourceConfig: {}, + syncIntervalMinutes: 1440, + membersBinding: { credentialGroupId: 'group-1', credentialGroupOptionId: 'option-1' }, + resolveBillingAttribution, + resolveAccessToken: vi.fn(), + ...ACTOR, + }) + + expect(outcome).toMatchObject({ success: true }) + expect(dbChainMockFns.for).toHaveBeenCalledWith('update') + expect(dbChainMockFns.insert).toHaveBeenCalledOnce() + expect(mockRevoke).not.toHaveBeenCalled() + }) + + it('refuses to create a members-mode connector on an option removed before the group was locked', async () => { + queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb-1' }]) + queueTableRows(schemaMock.credentialGroup, [{ options: [{ id: 'option-2' }] }]) + + const outcome = await performCreateKnowledgeConnector({ + knowledgeBase: KB, + connectorType: 'google_drive', + sourceConfig: {}, + syncIntervalMinutes: 1440, + membersBinding: { credentialGroupId: 'group-1', credentialGroupOptionId: 'option-1' }, + resolveBillingAttribution, + resolveAccessToken: vi.fn(), + ...ACTOR, + }) + + /** + * The group's option edit refuses while a connector row is bound to what it + * removes, and this row is written under the same lock, so the two can + * never both commit: the grant is undone and no row is inserted. + */ + expect(outcome).toMatchObject({ success: false, errorCode: 'validation' }) + expect(dbChainMockFns.insert).not.toHaveBeenCalled() + expect(mockRevoke).toHaveBeenCalledWith( + { workspaceId: 'ws-1', credentialGroupId: 'group-1', connectorId: expect.any(String) }, + 'user-1' + ) + }) + + it('refuses members mode for a connector whose listing is not permission scoped, before any grant', async () => { + queueTableRows(schemaMock.knowledgeBase, [{ id: 'kb-1' }]) + dbChainMockFns.returning.mockResolvedValueOnce([MEMBERS_CONNECTOR]) + + const outcome = await performCreateKnowledgeConnector({ + knowledgeBase: KB, + connectorType: 'notion', + sourceConfig: {}, + syncIntervalMinutes: 1440, + membersBinding: { credentialGroupId: 'group-1', credentialGroupOptionId: 'option-1' }, + resolveBillingAttribution, + resolveAccessToken: vi.fn(), + ...ACTOR, + }) + + expect(outcome).toMatchObject({ success: false, errorCode: 'validation' }) + expect(mockGrant).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/knowledge/orchestration/connectors.ts b/apps/sim/lib/knowledge/orchestration/connectors.ts index 5c08db52166..8c5596e220f 100644 --- a/apps/sim/lib/knowledge/orchestration/connectors.ts +++ b/apps/sim/lib/knowledge/orchestration/connectors.ts @@ -1,11 +1,13 @@ import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { db } from '@sim/db' import { + credentialGroup, document, embedding, knowledgeBase, knowledgeBaseTagDefinitions, knowledgeConnector, + knowledgeConnectorMember, } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { generateId } from '@sim/utils/id' @@ -15,6 +17,13 @@ import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attr import { hasWorkspaceLiveSyncAccess } from '@/lib/billing/core/subscription' import { OrchestrationError, type OrchestrationErrorCode } from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' +import type { DbOrTx } from '@/lib/db/types' +import { + findListingCapViolation, + grantKnowledgeConnectorCredentialAccess, + revokeKnowledgeConnectorCredentialAccess, + stripListingCapFields, +} from '@/lib/knowledge/connectors/member-access' import { allocateTagSlots } from '@/lib/knowledge/constants' import { deleteDocumentStorageFiles } from '@/lib/knowledge/documents/service' import { @@ -40,6 +49,51 @@ async function loadDispatchSync() { return (await import('@/lib/knowledge/connectors/queue')).dispatchSync } +async function loadDispatchMemberSync() { + return (await import('@/lib/knowledge/connectors/member-queue')).dispatchMemberSync +} + +/** The Credential Group option a members-mode connector crawls with, already validated by the caller. */ +export interface ConnectorMembersBinding { + credentialGroupId: string + credentialGroupOptionId: string +} + +/** + * Locks the Credential Group's row for the rest of the transaction and confirms + * the option is still part of it. The group's option edits and delete take the + * same row lock and refuse while a connector row is bound to what they remove, + * so a binding written under this lock is serialized against them: it either + * finds the option gone, or lands before the removal looks for it. The grant + * itself is a policy write with its own revision CAS, which is why the row + * write, not the grant, is what takes the lock. + */ +export async function lockCredentialGroupOption( + tx: DbOrTx, + binding: ConnectorMembersBinding & { workspaceId: string } +): Promise { + const [group] = await tx + .select({ options: credentialGroup.options }) + .from(credentialGroup) + .where( + and( + eq(credentialGroup.id, binding.credentialGroupId), + eq(credentialGroup.workspaceId, binding.workspaceId) + ) + ) + .limit(1) + .for('update') + if (!group) { + throw new OrchestrationError('validation', 'Credential Group was not found in this workspace') + } + if (!group.options.some((option) => option.id === binding.credentialGroupOptionId)) { + throw new OrchestrationError( + 'validation', + 'Credential option was not found in this Credential Group' + ) + } +} + /** A connector row exactly as stored, including its encrypted API key. */ export type KnowledgeConnectorRow = typeof knowledgeConnector.$inferSelect type ConnectorRow = KnowledgeConnectorRow @@ -87,6 +141,12 @@ export interface PerformCreateKnowledgeConnectorParams extends KnowledgeOperatio apiKey?: string sourceConfig: Record syncIntervalMinutes: number + /** + * Present when the connector crawls per member. The binding was validated + * against the group, the option, and the connector by the caller; the + * connector is granted the option's credentials before its row exists. + */ + membersBinding?: ConnectorMembersBinding /** * Resolves the payer the sync is billed to. A thunk so a request rejected by * a guard never pays for the lookup, and so the payer is read at the moment @@ -132,6 +192,7 @@ export async function performCreateKnowledgeConnector( apiKey, sourceConfig, syncIntervalMinutes, + membersBinding, resolveBillingAttribution, resolveAccessToken, request, @@ -158,9 +219,20 @@ export async function performCreateKnowledgeConnector( let resolvedCredentialId: string | null = null let resolvedEncryptedApiKey: string | null = null - let accessToken: string + let accessToken: string | null = null - if (connectorConfig.auth.mode === 'apiKey') { + if (membersBinding) { + /** + * A members-mode connector has no credential of its own to validate the + * source with: each member's first crawl validates it for that member. + * What can be checked here is that the config does not cap listings. + */ + if (connectorConfig.auth.mode !== 'oauth' || !connectorConfig.permissionScopedListing) { + return fail(`${connectorConfig.name} cannot sync per member`, 'validation') + } + const capViolation = findListingCapViolation(connectorConfig, sourceConfig) + if (capViolation) return fail(capViolation, 'validation') + } else if (connectorConfig.auth.mode === 'apiKey') { if (!apiKey && !connectorConfig.auth.optional) { return fail('API key is required', 'validation') } @@ -182,13 +254,15 @@ export async function performCreateKnowledgeConnector( resolvedCredentialId = credentialId } - const configValidation = await connectorConfig.validateConfig(accessToken, sourceConfig) - if (!configValidation.valid) { - return fail( - configValidation.error || - `The ${connectorType} connector rejected sourceConfig without a reason — re-check its required fields in knowledgebases/connectors/${connectorType}.json before retrying; the same config will fail again.`, - 'validation' - ) + if (accessToken !== null) { + const configValidation = await connectorConfig.validateConfig(accessToken, sourceConfig) + if (!configValidation.valid) { + return fail( + configValidation.error || + `The ${connectorType} connector rejected sourceConfig without a reason — re-check its required fields in knowledgebases/connectors/${connectorType}.json before retrying; the same config will fail again.`, + 'validation' + ) + } } if (connectorConfig.auth.mode === 'apiKey' && apiKey) { @@ -261,6 +335,27 @@ export async function performCreateKnowledgeConnector( const nextSyncAt = syncIntervalMinutes > 0 ? new Date(now.getTime() + syncIntervalMinutes * 60 * 1000) : null + /** + * Granted before the row exists so a connector can never be live without + * its grant; a failed insert revokes it again. The id is fixed above, so the + * policy names exactly the row about to be written. + */ + if (membersBinding) { + try { + await grantKnowledgeConnectorCredentialAccess( + { + workspaceId, + credentialGroupId: membersBinding.credentialGroupId, + credentialGroupOptionId: membersBinding.credentialGroupOptionId, + connectorId, + }, + params.userId + ) + } catch (error) { + return classifyKnowledgeFailure(error, requestId, `Create ${connectorType} connector`) + } + } + let created: ConnectorRow try { created = await db.transaction(async (tx) => { @@ -275,6 +370,9 @@ export async function performCreateKnowledgeConnector( if (activeKb.length === 0) { throw new OrchestrationError('not_found', 'Knowledge base not found') } + if (membersBinding) { + await lockCredentialGroupOption(tx, { workspaceId, ...membersBinding }) + } for (const [semanticId, slot] of Object.entries(newTagSlots)) { const td = connectorConfig.tagDefinitions?.find((d) => d.id === semanticId) @@ -309,9 +407,20 @@ export async function performCreateKnowledgeConnector( * rather than an idle connector until its first refetch. The lease and * ownership token that make the queue entry recoverable come from that * later write, which is why it must not skip an already-`pending` row. + * + * A members-mode connector is born `active`: its member run has its + * own queue state, and `pending` here would read as a content sync. */ - status: 'pending', - nextSyncAt, + status: membersBinding ? 'active' : 'pending', + nextSyncAt: membersBinding ? null : nextSyncAt, + ...(membersBinding + ? { + accessMode: 'members', + credentialGroupId: membersBinding.credentialGroupId, + credentialGroupOptionId: membersBinding.credentialGroupOptionId, + nextMemberSyncAt: now, + } + : {}), createdAt: now, updatedAt: now, }) @@ -320,6 +429,17 @@ export async function performCreateKnowledgeConnector( return row }) } catch (error) { + if (membersBinding) { + await revokeKnowledgeConnectorCredentialAccess( + { workspaceId, credentialGroupId: membersBinding.credentialGroupId, connectorId }, + params.userId + ).catch((revokeError) => { + logger.error(`[${requestId}] Failed to revoke the grant of an uncreated connector`, { + connectorId, + error: revokeError, + }) + }) + } return classifyKnowledgeFailure(error, requestId, `Create ${connectorType} connector`) } @@ -371,10 +491,15 @@ export async function performCreateKnowledgeConnector( * initial sync is at stake — so a failed enqueue is reported on the connector, * not by failing the creation. */ - const dispatchSync = await loadDispatchSync() let initialSyncQueued = true try { - const dispatch = await dispatchSync(connectorId, { billingAttribution, requestId }) + const dispatch = membersBinding + ? await (await loadDispatchMemberSync())(connectorId, { + billingAttribution, + requestId, + expectedNextMemberSyncAt: now, + }) + : await (await loadDispatchSync())(connectorId, { billingAttribution, requestId }) if (!dispatch.queued) { initialSyncQueued = false logger.warn( @@ -505,6 +630,24 @@ export async function performUpdateKnowledgeConnector( ) { return fail('Sync already in progress', 'conflict') } + /** + * A members-mode connector is run by the member engine, whose lease lives in + * `memberSyncStatus` while `status` stays `active`, so the two guards above + * never see it. The same two rules apply to that lease: a running member run + * owns the row, and a queued one has not read its config yet, so only a + * status change is safe. + */ + const syncsPerMember = existing.accessMode === 'members' + if (syncsPerMember && existing.memberSyncStatus === 'running') { + return fail('Sync already in progress', 'conflict') + } + if ( + syncsPerMember && + existing.memberSyncStatus === 'pending' && + (updates.sourceConfig !== undefined || updates.syncIntervalMinutes !== undefined) + ) { + return fail('Sync already in progress', 'conflict') + } if (updates.syncIntervalMinutes !== undefined) { if (!kb.workspaceId && updates.syncIntervalMinutes > 0 && updates.syncIntervalMinutes < 60) { @@ -519,10 +662,28 @@ export async function performUpdateKnowledgeConnector( } } - if (updates.sourceConfig !== undefined && validateSourceConfig) { - const rejection = await validateSourceConfig(existing, updates.sourceConfig) - if (rejection) { - return fail(rejection.message, rejection.errorCode) + let sourceConfigToStore = updates.sourceConfig + if (updates.sourceConfig !== undefined) { + if (existing.accessMode === 'members') { + /** + * A members-mode connector has no credential to validate the source + * with; the next member run does that per member. The listing caps are + * what a save can refuse. + */ + const { CONNECTOR_REGISTRY } = await import('@/connectors/registry.server') + const connectorConfig = CONNECTOR_REGISTRY[existing.connectorType] + const capViolation = connectorConfig + ? findListingCapViolation(connectorConfig, updates.sourceConfig) + : null + if (capViolation) return fail(capViolation, 'validation') + if (connectorConfig) { + sourceConfigToStore = stripListingCapFields(connectorConfig, updates.sourceConfig) + } + } else if (validateSourceConfig) { + const rejection = await validateSourceConfig(existing, updates.sourceConfig) + if (rejection) { + return fail(rejection.message, rejection.errorCode) + } } } @@ -531,12 +692,22 @@ export async function performUpdateKnowledgeConnector( updates.sourceConfig !== undefined && resultingStatus !== 'paused' && resultingStatus !== 'disabled' + /** + * The schedule this connector is picked up by: the member scheduler reads + * `nextMemberSyncAt` and the content scheduler `nextSyncAt`, each only for + * its own access mode, so every schedule write below lands on the one the + * connector's engine will read. + */ + const scheduleColumn = syncsPerMember ? 'nextMemberSyncAt' : 'nextSyncAt' + const existingSchedule = existing[scheduleColumn] let billingAttribution: BillingAttributionSnapshot | undefined let dispatchSourceSync: Awaited> | undefined + let dispatchMemberSourceSync: Awaited> | undefined if (shouldDispatchSourceSync) { try { billingAttribution = await resolveBillingAttribution() - dispatchSourceSync = await loadDispatchSync() + if (syncsPerMember) dispatchMemberSourceSync = await loadDispatchMemberSync() + else dispatchSourceSync = await loadDispatchSync() } catch (error) { return classifyKnowledgeFailure(error, requestId, `Update connector ${connectorId}`) } @@ -546,14 +717,14 @@ export async function performUpdateKnowledgeConnector( const values: Partial = { updatedAt: updateTimestamp, } - if (updates.sourceConfig !== undefined) { - values.sourceConfig = updates.sourceConfig + if (sourceConfigToStore !== undefined) { + values.sourceConfig = sourceConfigToStore } if (updates.syncIntervalMinutes !== undefined) { values.syncIntervalMinutes = updates.syncIntervalMinutes - values.nextSyncAt = - existing.nextSyncAt && existing.nextSyncAt <= updateTimestamp - ? existing.nextSyncAt + values[scheduleColumn] = + existingSchedule && existingSchedule <= updateTimestamp + ? existingSchedule : updates.syncIntervalMinutes > 0 ? new Date(updateTimestamp.getTime() + updates.syncIntervalMinutes * 60 * 1000) : null @@ -563,24 +734,32 @@ export async function performUpdateKnowledgeConnector( /** * Releases a queue entry this status change is walking away from, so no * token survives on a row that is no longer `pending` and the reaper is not - * left with a lease it can never match. + * left with a lease it can never match. A queued member run is released the + * same way: its task starts without re-checking `status`, so the entry has + * to be gone for a pause to hold, and the CAS below keeps this off a run + * that has since started. */ if (existing.status === 'pending') { values.syncLockToken = null values.syncLockLeaseAt = null } + if (syncsPerMember && existing.memberSyncStatus === 'pending') { + values.memberSyncStatus = 'idle' + values.memberSyncLockToken = null + values.memberSyncLockLeaseAt = null + } if (updates.status === 'active') { values.consecutiveFailures = 0 values.lastSyncError = null // Resuming a paused connector syncs immediately unless this same request // set a schedule, which then owns the next run. - if (values.nextSyncAt === undefined) { - values.nextSyncAt = new Date() + if (values[scheduleColumn] === undefined) { + values[scheduleColumn] = new Date() } } } if (shouldDispatchSourceSync) { - values.nextSyncAt = updateTimestamp + values[scheduleColumn] = updateTimestamp } let updated: ConnectorRow @@ -592,11 +771,14 @@ export async function performUpdateKnowledgeConnector( isNull(knowledgeConnector.deletedAt), ] updateConditions.push(eq(knowledgeConnector.status, existing.status)) - if (values.nextSyncAt !== undefined) { + if (syncsPerMember) { + updateConditions.push(eq(knowledgeConnector.memberSyncStatus, existing.memberSyncStatus)) + } + if (values[scheduleColumn] !== undefined) { updateConditions.push( - existing.nextSyncAt - ? eq(knowledgeConnector.nextSyncAt, existing.nextSyncAt) - : isNull(knowledgeConnector.nextSyncAt) + existingSchedule + ? eq(knowledgeConnector[scheduleColumn], existingSchedule) + : isNull(knowledgeConnector[scheduleColumn]) ) } @@ -608,7 +790,7 @@ export async function performUpdateKnowledgeConnector( if (!row) { const current = await getKnowledgeConnector(kb.id, connectorId) - if (current?.status === 'syncing') { + if (current?.status === 'syncing' || current?.memberSyncStatus === 'running') { return fail('Sync already in progress', 'conflict') } if (current) { @@ -645,6 +827,23 @@ export async function performUpdateKnowledgeConnector( }) } + if (dispatchMemberSourceSync && billingAttribution) { + try { + await dispatchMemberSourceSync(connectorId, { + billingAttribution, + expectedNextMemberSyncAt: updateTimestamp, + requestId, + requireRunnable: true, + }) + } catch (error) { + return classifyKnowledgeFailure( + error, + requestId, + `Dispatch source-change member sync for connector ${connectorId}` + ) + } + } + if (dispatchSourceSync && billingAttribution) { try { await dispatchSourceSync(connectorId, { @@ -705,6 +904,17 @@ export async function performDeleteKnowledgeConnector( if (!existing) { return fail('Connector not found', 'not_found') } + /** + * A members-mode document's visibility is its observers; detached from the + * connector it would keep an ACL nothing maintains, or become hidden to + * everyone. Neither is a standalone entry anyone asked for. + */ + if (existing.accessMode === 'members' && !deleteDocuments) { + return fail( + 'Documents of a connector that syncs per member cannot be kept; delete them with the connector', + 'conflict' + ) + } let deletedDocs: Array<{ id: string; fileUrl: string }> let docCount: number @@ -769,6 +979,22 @@ export async function performDeleteKnowledgeConnector( ]) } + if (existing.credentialGroupId && kb.workspaceId) { + await revokeKnowledgeConnectorCredentialAccess( + { + workspaceId: kb.workspaceId, + credentialGroupId: existing.credentialGroupId, + connectorId, + }, + params.userId + ).catch((error) => { + logger.error(`[${requestId}] Failed to revoke the deleted connector's credential access`, { + connectorId, + error, + }) + }) + } + logger.info( `[${requestId}] Deleted connector ${connectorId}${deleteDocuments ? ` and ${docCount} documents` : `, kept ${docCount} documents`}` ) @@ -849,6 +1075,15 @@ export async function performSyncKnowledgeConnector( if (connector.status === 'syncing' || connector.status === 'pending') { return fail('Sync already in progress', 'conflict') } + if (connector.memberSyncStatus === 'running' || connector.memberSyncStatus === 'pending') { + return fail('Sync already in progress', 'conflict') + } + if (connector.accessMode === 'members' && connector.memberSyncStatus === 'disabled') { + return fail( + connector.lastMemberSyncError ?? 'Member sync is disabled for this connector', + 'conflict' + ) + } /** * A paused or disabled connector is not synced on demand. * @@ -874,6 +1109,12 @@ export async function performSyncKnowledgeConnector( return classifyKnowledgeFailure(error, requestId, `Sync connector ${connectorId}`) } + if (rehydrate && connector.accessMode === 'members') { + return fail( + 'A connector that syncs per member re-hydrates through its members; run a sync instead', + 'validation' + ) + } logger.info( `[${requestId}] Manual sync${rehydrate ? ' (full rehydrate)' : ''} triggered for connector ${connectorId}` ) @@ -886,9 +1127,31 @@ export async function performSyncKnowledgeConnector( * product event for work that never started. Awaiting first makes the reported * outcome and both records describe what actually happened. */ - const dispatchSync = await loadDispatchSync() try { - const dispatch = await dispatchSync(connectorId, { billingAttribution, requestId, rehydrate }) + /** + * A manual run is meant to list everyone now, so every active member is + * made due; otherwise each waits out its own interval and the run claims + * nobody. + */ + if (connector.accessMode === 'members') { + await db + .update(knowledgeConnectorMember) + .set({ nextAttemptAt: new Date(), updatedAt: new Date() }) + .where( + and( + eq(knowledgeConnectorMember.connectorId, connectorId), + eq(knowledgeConnectorMember.status, 'active') + ) + ) + } + const dispatch = + connector.accessMode === 'members' + ? await (await loadDispatchMemberSync())(connectorId, { billingAttribution, requestId }) + : await (await loadDispatchSync())(connectorId, { + billingAttribution, + requestId, + rehydrate, + }) /** * A guard inside the dispatch declining to queue is reported as a failure * rather than a queued sync. Every one of them means the connector's state diff --git a/apps/sim/lib/knowledge/search/author.test.ts b/apps/sim/lib/knowledge/search/author.test.ts new file mode 100644 index 00000000000..e3a9098f505 --- /dev/null +++ b/apps/sim/lib/knowledge/search/author.test.ts @@ -0,0 +1,23 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { sourceAuthor } from '@/lib/knowledge/search/author' + +describe('sourceAuthor', () => { + it('prefers the sender of an email and drops the address', () => { + expect(sourceAuthor({ From: '"Ada Lovelace" ', Owner: 'Someone' })).toBe( + 'Ada Lovelace' + ) + }) + + it('falls through the author-like names in order', () => { + expect(sourceAuthor({ Assignee: 'Grace', Owner: 'Alan' })).toBe('Alan') + expect(sourceAuthor({ Reporter: 'Grace' })).toBe('Grace') + }) + + it('returns null when nothing names a person', () => { + expect(sourceAuthor({ From: '', Status: 'Open' })).toBeNull() + expect(sourceAuthor({})).toBeNull() + }) +}) diff --git a/apps/sim/lib/knowledge/search/author.ts b/apps/sim/lib/knowledge/search/author.ts new file mode 100644 index 00000000000..bd6bff80d5e --- /dev/null +++ b/apps/sim/lib/knowledge/search/author.ts @@ -0,0 +1,34 @@ +/** + * The tag names connectors give the person behind a document, in the order + * they are tried. Connectors were never asked to agree on a name, so the + * result's author is derived here rather than in each of them. + */ +const AUTHOR_TAG_NAMES = [ + 'From', + 'Author', + 'Sender', + 'Owner', + 'Organizer', + 'Creator', + 'Reporter', + 'Assignee', +] as const + +/** + * The person a search result shows beside its source: the first author-like + * tag the document carries, reduced to a display name when the connector + * stored an address form such as `Name `. + */ +export function sourceAuthor(metadata: Record): string | null { + for (const name of AUTHOR_TAG_NAMES) { + const value = metadata[name] + if (typeof value !== 'string') continue + const display = value + .replace(/<[^>]*>/g, '') + .trim() + .replace(/^"|"$/g, '') + .trim() + if (display) return display + } + return null +} diff --git a/apps/sim/lib/knowledge/search/defaults.test.ts b/apps/sim/lib/knowledge/search/defaults.test.ts new file mode 100644 index 00000000000..0cac51ff9c1 --- /dev/null +++ b/apps/sim/lib/knowledge/search/defaults.test.ts @@ -0,0 +1,72 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockAvailable } = vi.hoisted(() => ({ mockAvailable: vi.fn() })) + +vi.mock('@/lib/knowledge/access/availability', () => ({ + isKnowledgeMemberAccessAvailable: mockAvailable, +})) + +import { resolveKnowledgeSearchDefaults } from '@/lib/knowledge/search/defaults' + +describe('resolveKnowledgeSearchDefaults', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('stays semantic-only with no boost where the feature is off', async () => { + mockAvailable.mockResolvedValue(false) + await expect( + resolveKnowledgeSearchDefaults({ + workspaceId: 'ws-1', + userId: 'u-1', + requestedMode: undefined, + }) + ).resolves.toEqual({ searchMode: 'vector', boostRecency: false }) + expect(mockAvailable).toHaveBeenCalledWith({ workspaceId: 'ws-1', userId: 'u-1' }) + }) + + it('defaults to hybrid with the recency boost where the feature is on', async () => { + mockAvailable.mockResolvedValue(true) + await expect( + resolveKnowledgeSearchDefaults({ + workspaceId: 'ws-1', + userId: undefined, + requestedMode: undefined, + }) + ).resolves.toEqual({ searchMode: 'hybrid', boostRecency: true }) + expect(mockAvailable).toHaveBeenCalledWith({ workspaceId: 'ws-1', userId: undefined }) + }) + + it('keeps an explicit mode either way', async () => { + mockAvailable.mockResolvedValue(true) + await expect( + resolveKnowledgeSearchDefaults({ + workspaceId: 'ws-1', + userId: 'u-1', + requestedMode: 'vector', + }) + ).resolves.toEqual({ searchMode: 'vector', boostRecency: true }) + mockAvailable.mockResolvedValue(false) + await expect( + resolveKnowledgeSearchDefaults({ + workspaceId: 'ws-1', + userId: 'u-1', + requestedMode: 'hybrid', + }) + ).resolves.toEqual({ searchMode: 'hybrid', boostRecency: false }) + }) + + it('never consults the flag without a workspace, and reads as off', async () => { + await expect( + resolveKnowledgeSearchDefaults({ + workspaceId: undefined, + userId: 'u-1', + requestedMode: undefined, + }) + ).resolves.toEqual({ searchMode: 'vector', boostRecency: false }) + expect(mockAvailable).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/knowledge/search/defaults.ts b/apps/sim/lib/knowledge/search/defaults.ts new file mode 100644 index 00000000000..9bff2b2f281 --- /dev/null +++ b/apps/sim/lib/knowledge/search/defaults.ts @@ -0,0 +1,33 @@ +import { isKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' +import type { KnowledgeSearchMode } from '@/lib/knowledge/search/queries' + +/** How a search runs when the caller did not choose a mode. */ +export interface KnowledgeSearchDefaults { + searchMode: KnowledgeSearchMode + /** Whether a recently modified document may edge past a stale one of similar relevance. */ + boostRecency: boolean +} + +/** + * The retrieval defaults for one workspace. Where permission-aware knowledge + * is on, hybrid retrieval is the default and every search gets the recency + * boost; elsewhere search stays semantic-only with no boost, exactly as + * before. An explicit `searchMode` from the caller always wins over the + * default mode; the boost is a workspace policy and applies to either mode. + */ +export async function resolveKnowledgeSearchDefaults(input: { + workspaceId: string | undefined + userId: string | undefined + requestedMode: KnowledgeSearchMode | undefined +}): Promise { + const enabled = input.workspaceId + ? await isKnowledgeMemberAccessAvailable({ + workspaceId: input.workspaceId, + userId: input.userId, + }) + : false + return { + searchMode: input.requestedMode ?? (enabled ? 'hybrid' : 'vector'), + boostRecency: enabled, + } +} diff --git a/apps/sim/lib/knowledge/search/queries.ts b/apps/sim/lib/knowledge/search/queries.ts index 9136a80a9d8..57cece6f4c6 100644 --- a/apps/sim/lib/knowledge/search/queries.ts +++ b/apps/sim/lib/knowledge/search/queries.ts @@ -1,8 +1,11 @@ import { db } from '@sim/db' -import { document, embedding } from '@sim/db/schema' +import { document, embedding, knowledgeConnector } from '@sim/db/schema' import { createLogger } from '@sim/logger' -import { getErrorMessage } from '@sim/utils/errors' +import { getErrorMessage, getPostgresErrorCode } from '@sim/utils/errors' import { and, eq, inArray, isNull, type SQL, sql } from 'drizzle-orm' +import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' +import { type KnowledgeAccessScope, WORKSPACE_ACCESS_TOKENS } from '@/lib/knowledge/access/types' +import { applyRecencyBoost, RRF_K } from '@/lib/knowledge/search/recency' import { coerceTagFilterValue, escapeLikePattern, @@ -12,21 +15,84 @@ import type { StructuredFilter } from '@/lib/knowledge/types' const logger = createLogger('KnowledgeSearchQueries') +/** SQLSTATE for an unrecognised configuration parameter — pgvector older than 0.8. */ +const UNDEFINED_OBJECT_SQLSTATE = '42704' +/** Tuples a relaxed-order scan may visit before giving up on filling the limit. */ +const HNSW_MAX_SCAN_TUPLES = '20000' +/** pgvector's default `hnsw.ef_search`: the candidates a plain scan yields before predicates. */ +const HNSW_DEFAULT_EF_SEARCH = 40 + +/** How long to stop trying the iterative-scan settings after the server rejected them. */ +const HNSW_SETTINGS_UNSUPPORTED_RETRY_MS = 10 * 60 * 1000 + +let hnswSettingsUnsupportedUntil = 0 + +type SearchExecutor = Pick + +/** + * Runs a vector leg with pgvector's iterative HNSW scan enabled. A plain scan + * yields at most `hnsw.ef_search` candidates before the access predicate is + * applied, so a caller who may see a small share of a base gets fewer rows + * than asked for; `relaxed_order` keeps scanning until the limit is met. The + * settings are transaction-local, which needs a transaction: under PgBouncer + * transaction pooling a bare `SET LOCAL` is a no-op. Servers without the + * setting (pgvector < 0.8) reject it with 42704; the leg then runs unscoped + * and the attempt is retried after a while so an upgrade is picked up. + */ +async function withVectorScanSettings( + access: KnowledgeAccessScope, + limit: number, + run: (executor: SearchExecutor) => Promise +): Promise { + /** + * A plain index scan yields `hnsw.ef_search` candidates (40 by default) + * before the predicates apply. That fills a small limit for the workspace + * pair, which matches every row; a personal token set, or a limit past the + * pool, needs the iterative scan to keep going until the limit is met. + */ + const needsIterativeScan = hasSubjectTokens(access) || limit > HNSW_DEFAULT_EF_SEARCH + if (!needsIterativeScan || Date.now() < hnswSettingsUnsupportedUntil) return run(db) + try { + return await db.transaction(async (tx) => { + await tx.execute( + sql`SELECT set_config('hnsw.iterative_scan', 'relaxed_order', true), set_config('hnsw.max_scan_tuples', ${HNSW_MAX_SCAN_TUPLES}, true)` + ) + return run(tx) + }) + } catch (error) { + if (getPostgresErrorCode(error) !== UNDEFINED_OBJECT_SQLSTATE) throw error + hnswSettingsUnsupportedUntil = Date.now() + HNSW_SETTINGS_UNSUPPORTED_RETRY_MS + logger.warn('pgvector iterative scan is unavailable; vector legs run without it', { + error: getErrorMessage(error), + }) + return run(db) + } +} + +/** Whether the caller holds tokens beyond the workspace pair every document carries. */ +function hasSubjectTokens(access: KnowledgeAccessScope): boolean { + return access.kind === 'user' && access.tokens.length > WORKSPACE_ACCESS_TOKENS.length +} + export interface DocumentMetadata { filename: string sourceUrl: string | null + /** When the source last changed the document; null for uploads and sources that do not say. */ + sourceModifiedAt: Date | null + /** The connector the document was synced through; null for an upload. */ + connectorType: string | null } /** * Batch-fetch display metadata for documents referenced by search results. - * Excludes documents that are user-excluded, archived, or soft-deleted — - * mirrors the visibility filters applied inside the search SQL itself, so - * the lookup will never surface metadata for a row a caller could not have - * legitimately matched. Returns a map keyed by document id; missing ids - * indicate the document is no longer visible and should be skipped. + * Applies the same visibility and access predicates as the search SQL itself, + * so the lookup never surfaces a filename for a row the caller could not have + * matched. Returns a map keyed by document id; missing ids indicate the + * document is no longer visible and should be skipped. */ export async function getDocumentMetadataByIds( - documentIds: string[] + documentIds: string[], + access: KnowledgeAccessScope ): Promise> { if (documentIds.length === 0) { return {} @@ -38,20 +104,29 @@ export async function getDocumentMetadataByIds( id: document.id, filename: document.filename, sourceUrl: document.sourceUrl, + sourceModifiedAt: document.sourceModifiedAt, + connectorType: knowledgeConnector.connectorType, }) .from(document) + .leftJoin(knowledgeConnector, eq(knowledgeConnector.id, document.connectorId)) .where( and( inArray(document.id, uniqueIds), eq(document.userExcluded, false), isNull(document.archivedAt), - isNull(document.deletedAt) + isNull(document.deletedAt), + knowledgeAccessCondition(access) ) ) const map: Record = {} documents.forEach((doc) => { - map[doc.id] = { filename: doc.filename, sourceUrl: doc.sourceUrl ?? null } + map[doc.id] = { + filename: doc.filename, + sourceUrl: doc.sourceUrl ?? null, + sourceModifiedAt: doc.sourceModifiedAt ?? null, + connectorType: doc.connectorType ?? null, + } }) return map @@ -85,18 +160,20 @@ export interface SearchResult { boolean3: boolean | null distance: number knowledgeBaseId: string + /** When the source last changed the document; NULL for uploads and sources that do not say. */ + sourceModifiedAt: Date | null } export interface SearchParams { knowledgeBaseIds: string[] topK: number + /** What the caller may read; every leg applies it. Required so no leg can be written without it. */ + access: KnowledgeAccessScope structuredFilters?: StructuredFilter[] queryVector?: string distanceThreshold?: number } -export { generateSearchEmbedding } from '@/lib/knowledge/embeddings' - /** All valid tag slot keys */ const TAG_SLOT_KEYS = [ // Text tags (7 slots) @@ -157,6 +234,7 @@ const getSearchResultFields = (distanceExpr: any) => ({ boolean3: embedding.boolean3, distance: distanceExpr, knowledgeBaseId: embedding.knowledgeBaseId, + sourceModifiedAt: document.sourceModifiedAt, }) /** @@ -312,18 +390,14 @@ export function getStructuredTagFilters(filters: StructuredFilter[], embeddingTa */ const FTS_CONFIG = 'english' -/** - * Reciprocal-rank-fusion damping constant. 60 is the value from the original RRF - * paper and matches the docs search retriever (`apps/docs/app/api/search/route.ts`). - */ -export const RRF_K = 60 - /** * Row visibility predicates shared by every search leg: a chunk is only * retrievable when both it and its document are enabled, the document finished - * processing, and it has not been excluded, archived, or soft-deleted. + * processing, it has not been excluded, archived, or soft-deleted, and its ACL + * overlaps the caller's tokens. Every leg spreads this helper rather than + * listing the predicates itself, so no leg can drift from the others. */ -function getVisibilityConditions() { +function getVisibilityConditions(access: KnowledgeAccessScope) { return [ eq(embedding.enabled, true), eq(document.enabled, true), @@ -331,9 +405,17 @@ function getVisibilityConditions() { eq(document.userExcluded, false), isNull(document.archivedAt), isNull(document.deletedAt), + knowledgeAccessCondition(access), ] } +/** Candidates each hybrid leg retrieves before the fused list is trimmed to `topK`. */ +const HYBRID_CANDIDATE_MIN = 50 +const HYBRID_CANDIDATE_MAX = 200 +export function hybridCandidateCount(topK: number): number { + return Math.min(Math.max(topK * 3, HYBRID_CANDIDATE_MIN), HYBRID_CANDIDATE_MAX) +} + export function getQueryStrategy(kbCount: number, topK: number) { const useParallel = kbCount > 4 || (kbCount > 2 && topK > 50) const distanceThreshold = kbCount > 3 ? 0.8 : 1.0 @@ -349,81 +431,57 @@ export function getQueryStrategy(kbCount: number, topK: number) { async function executeTagFilterQuery( knowledgeBaseIds: string[], - structuredFilters: StructuredFilter[] + structuredFilters: StructuredFilter[], + access: KnowledgeAccessScope ): Promise<{ id: string }[]> { const tagFilterConditions = getStructuredTagFilters(structuredFilters, embedding) + const kbScope = + knowledgeBaseIds.length === 1 + ? eq(embedding.knowledgeBaseId, knowledgeBaseIds[0]) + : inArray(embedding.knowledgeBaseId, knowledgeBaseIds) - if (knowledgeBaseIds.length === 1) { - return await db - .select({ id: embedding.id }) - .from(embedding) - .innerJoin(document, eq(embedding.documentId, document.id)) - .where( - and( - eq(embedding.knowledgeBaseId, knowledgeBaseIds[0]), - eq(embedding.enabled, true), - eq(document.enabled, true), - eq(document.processingStatus, 'completed'), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt), - ...tagFilterConditions - ) - ) - } return await db .select({ id: embedding.id }) .from(embedding) .innerJoin(document, eq(embedding.documentId, document.id)) - .where( - and( - inArray(embedding.knowledgeBaseId, knowledgeBaseIds), - eq(embedding.enabled, true), - eq(document.enabled, true), - eq(document.processingStatus, 'completed'), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt), - ...tagFilterConditions - ) - ) + .where(and(kbScope, ...getVisibilityConditions(access), ...tagFilterConditions)) } async function executeVectorSearchOnIds( embeddingIds: string[], queryVector: string, topK: number, - distanceThreshold: number + distanceThreshold: number, + access: KnowledgeAccessScope ): Promise { if (embeddingIds.length === 0) { return [] } - return await db - .select( - getSearchResultFields( - sql`${embedding.embedding} <=> ${queryVector}::vector`.as('distance') + const rows = await withVectorScanSettings(access, topK, (executor) => + executor + .select( + getSearchResultFields( + sql`${embedding.embedding} <=> ${queryVector}::vector`.as('distance') + ) ) - ) - .from(embedding) - .innerJoin(document, eq(embedding.documentId, document.id)) - .where( - and( - inArray(embedding.id, embeddingIds), - eq(document.enabled, true), - eq(document.processingStatus, 'completed'), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt), - sql`${embedding.embedding} <=> ${queryVector}::vector < ${distanceThreshold}` + .from(embedding) + .innerJoin(document, eq(embedding.documentId, document.id)) + .where( + and( + inArray(embedding.id, embeddingIds), + ...getVisibilityConditions(access), + sql`${embedding.embedding} <=> ${queryVector}::vector < ${distanceThreshold}` + ) ) - ) - .orderBy(sql`${embedding.embedding} <=> ${queryVector}::vector`) - .limit(topK) + .orderBy(sql`${embedding.embedding} <=> ${queryVector}::vector`) + .limit(topK) + ) + return rows.sort((a, b) => a.distance - b.distance) } export async function handleTagOnlySearch(params: SearchParams): Promise { - const { knowledgeBaseIds, topK, structuredFilters } = params + const { knowledgeBaseIds, topK, structuredFilters, access } = params if (!structuredFilters || structuredFilters.length === 0) { throw new Error('Tag filters are required for tag-only search') @@ -443,12 +501,7 @@ export async function handleTagOnlySearch(params: SearchParams): Promise { - const { knowledgeBaseIds, topK, queryVector, distanceThreshold } = params + const { knowledgeBaseIds, topK, queryVector, distanceThreshold, access } = params if (!queryVector || !distanceThreshold) { throw new Error('Query vector and distance threshold are required for vector-only search') @@ -488,59 +536,47 @@ export async function handleVectorOnlySearch(params: SearchParams): Promise`${embedding.embedding} <=> ${queryVector}::vector`.as('distance') + const vectorLeg = (executor: SearchExecutor, kbScope: SQL | undefined, limit: number) => + executor + .select(getSearchResultFields(distanceExpr)) + .from(embedding) + .innerJoin(document, eq(embedding.documentId, document.id)) + .where( + and( + kbScope, + ...getVisibilityConditions(access), + sql`${embedding.embedding} <=> ${queryVector}::vector < ${distanceThreshold}` + ) + ) + .orderBy(sql`${embedding.embedding} <=> ${queryVector}::vector`) + .limit(limit) + /** + * A relaxed-order iterative scan may hand rows back slightly out of distance + * order, so both paths re-sort in memory before trimming to `topK`. + */ if (strategy.useParallel) { const parallelLimit = Math.ceil(topK / knowledgeBaseIds.length) + 5 - - const queryPromises = knowledgeBaseIds.map(async (kbId) => { - return await db - .select(getSearchResultFields(distanceExpr)) - .from(embedding) - .innerJoin(document, eq(embedding.documentId, document.id)) - .where( - and( - eq(embedding.knowledgeBaseId, kbId), - eq(embedding.enabled, true), - eq(document.enabled, true), - eq(document.processingStatus, 'completed'), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt), - sql`${embedding.embedding} <=> ${queryVector}::vector < ${distanceThreshold}` - ) + const allResults = await withVectorScanSettings(access, parallelLimit, async (executor) => { + const parallelResults = await Promise.all( + knowledgeBaseIds.map((kbId) => + vectorLeg(executor, eq(embedding.knowledgeBaseId, kbId), parallelLimit) ) - .orderBy(sql`${embedding.embedding} <=> ${queryVector}::vector`) - .limit(parallelLimit) + ) + return parallelResults.flat() }) - - const parallelResults = await Promise.all(queryPromises) - const allResults = parallelResults.flat() return allResults.sort((a, b) => a.distance - b.distance).slice(0, topK) } - // Single query for fewer KBs - return await db - .select(getSearchResultFields(distanceExpr)) - .from(embedding) - .innerJoin(document, eq(embedding.documentId, document.id)) - .where( - and( - inArray(embedding.knowledgeBaseId, knowledgeBaseIds), - eq(embedding.enabled, true), - eq(document.enabled, true), - eq(document.processingStatus, 'completed'), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt), - sql`${embedding.embedding} <=> ${queryVector}::vector < ${distanceThreshold}` - ) - ) - .orderBy(sql`${embedding.embedding} <=> ${queryVector}::vector`) - .limit(topK) + const rows = await withVectorScanSettings(access, topK, (executor) => + vectorLeg(executor, inArray(embedding.knowledgeBaseId, knowledgeBaseIds), topK) + ) + return rows.sort((a, b) => a.distance - b.distance) } export interface KeywordSearchParams { knowledgeBaseIds: string[] topK: number + access: KnowledgeAccessScope query: string /** Query embedding, so keyword-only hits still carry a real cosine distance. */ queryVector: string @@ -572,7 +608,7 @@ export interface KeywordSearchParams { * that survive the limit are hydrated. */ export async function executeKeywordSearch(params: KeywordSearchParams): Promise { - const { knowledgeBaseIds, topK, query, queryVector, structuredFilters } = params + const { knowledgeBaseIds, topK, query, queryVector, structuredFilters, access } = params if (!query.trim()) { return [] @@ -587,7 +623,7 @@ export async function executeKeywordSearch(params: KeywordSearchParams): Promise const rankConditions = (kbScope: SQL | undefined) => and( kbScope, - ...getVisibilityConditions(), + ...getVisibilityConditions(access), sql`${embedding.contentTsv} @@ ${tsQuery}`, ...tagFilterConditions ) @@ -629,7 +665,7 @@ export async function executeKeywordSearch(params: KeywordSearchParams): Promise ) .from(embedding) .innerJoin(document, eq(embedding.documentId, document.id)) - .where(and(inArray(embedding.id, topIds), ...getVisibilityConditions())) + .where(and(inArray(embedding.id, topIds), ...getVisibilityConditions(access))) const rowById = new Map(hydrated.map((row) => [row.id, row])) return topIds.map((id) => rowById.get(id)).filter((row): row is SearchResult => row !== undefined) @@ -718,7 +754,8 @@ export function fuseByReciprocalRank(rankedLists: SearchResult[][], topK: number } export async function handleTagAndVectorSearch(params: SearchParams): Promise { - const { knowledgeBaseIds, topK, structuredFilters, queryVector, distanceThreshold } = params + const { knowledgeBaseIds, topK, structuredFilters, queryVector, distanceThreshold, access } = + params if (!structuredFilters || structuredFilters.length === 0) { throw new Error('Tag filters are required for tag and vector search') @@ -727,7 +764,7 @@ export async function handleTagAndVectorSearch(params: SearchParams): Promise r.id), queryVector, topK, - distanceThreshold + distanceThreshold, + access ) } @@ -751,7 +789,11 @@ export interface ExecuteKnowledgeSearchParams { knowledgeBaseIds: string[] /** Candidate count each leg retrieves and the fused list is trimmed to. */ topK: number + /** What the caller may read; resolved from the principal by the use case, never from input. */ + access: KnowledgeAccessScope searchMode: KnowledgeSearchMode + /** Lets a recently modified document edge past a stale one of similar relevance; off by default. */ + boostRecency?: boolean query?: string /** Required whenever `query` is present. */ queryVector?: string @@ -766,7 +808,16 @@ export interface ExecuteKnowledgeSearchParams { export async function executeKnowledgeSearch( params: ExecuteKnowledgeSearchParams ): Promise { - const { knowledgeBaseIds, topK, searchMode, query, queryVector, structuredFilters } = params + const { + knowledgeBaseIds, + topK, + searchMode, + query, + queryVector, + structuredFilters, + access, + boostRecency = false, + } = params const hasQuery = Boolean(query?.trim()) const hasFilters = Boolean(structuredFilters && structuredFilters.length > 0) @@ -775,7 +826,7 @@ export async function executeKnowledgeSearch( if (!hasFilters) { throw new Error('A search query or tag filters are required') } - return await handleTagOnlySearch({ knowledgeBaseIds, topK, structuredFilters }) + return await handleTagOnlySearch({ knowledgeBaseIds, topK, structuredFilters, access }) } if (!queryVector) { @@ -783,19 +834,33 @@ export async function executeKnowledgeSearch( } const { distanceThreshold } = getQueryStrategy(knowledgeBaseIds.length, topK) + /** + * Hybrid fuses two rankings, so each leg retrieves more than the caller + * asked for: a chunk that both legs rank just below `topK` is a strong + * signal the fused list must be able to surface. + */ + const legTopK = searchMode === 'hybrid' ? hybridCandidateCount(topK) : topK const vectorSearch = hasFilters ? handleTagAndVectorSearch({ knowledgeBaseIds, - topK, + topK: legTopK, structuredFilters, queryVector, distanceThreshold, + access, + }) + : handleVectorOnlySearch({ + knowledgeBaseIds, + topK: legTopK, + queryVector, + distanceThreshold, + access, }) - : handleVectorOnlySearch({ knowledgeBaseIds, topK, queryVector, distanceThreshold }) if (searchMode === 'vector') { - return await vectorSearch + const results = await vectorSearch + return boostRecency ? applyRecencyBoost(results) : results } /** @@ -804,10 +869,11 @@ export async function executeKnowledgeSearch( */ const keywordSearch = executeKeywordSearch({ knowledgeBaseIds, - topK, + topK: legTopK, query: query!, queryVector, structuredFilters, + access, }).catch((error) => { logger.warn('Keyword search leg failed; falling back to vector-only results', { error: getErrorMessage(error, 'Unknown error'), @@ -823,5 +889,6 @@ export async function executeKnowledgeSearch( * threshold is precisely what a caller opted into hybrid to recover, and at * `topK: 1` something has to win. */ - return fuseByReciprocalRank([keywordResults, vectorResults], topK) + const fused = fuseByReciprocalRank([keywordResults, vectorResults], topK) + return boostRecency ? applyRecencyBoost(fused) : fused } diff --git a/apps/sim/lib/knowledge/search/recency.test.ts b/apps/sim/lib/knowledge/search/recency.test.ts new file mode 100644 index 00000000000..ae6791a4fc0 --- /dev/null +++ b/apps/sim/lib/knowledge/search/recency.test.ts @@ -0,0 +1,57 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + applyRecencyBoost, + RECENCY_HALF_LIFE_DAYS, + recencyFreshness, +} from '@/lib/knowledge/search/recency' + +const NOW = new Date('2026-09-01T12:00:00Z') +const DAY_MS = 24 * 60 * 60 * 1000 + +function daysAgo(days: number): Date { + return new Date(NOW.getTime() - days * DAY_MS) +} + +describe('recencyFreshness', () => { + it('is 1 now, halves at the half-life, and is 0 without a modified time', () => { + expect(recencyFreshness(NOW, NOW)).toBe(1) + expect(recencyFreshness(daysAgo(RECENCY_HALF_LIFE_DAYS), NOW)).toBeCloseTo(0.5) + expect(recencyFreshness(null, NOW)).toBe(0) + expect(recencyFreshness(undefined, NOW)).toBe(0) + }) + + it('gives a future-dated document no boost rather than an inflated one', () => { + expect(recencyFreshness(daysAgo(-1), NOW)).toBe(0) + }) +}) + +describe('applyRecencyBoost', () => { + const row = (id: string, sourceModifiedAt: Date | null) => ({ id, sourceModifiedAt }) + + it('keeps rank order when nothing carries a modified time', () => { + const rows = [row('a', null), row('b', null), row('c', null)] + expect(applyRecencyBoost(rows, NOW).map((r) => r.id)).toEqual(['a', 'b', 'c']) + }) + + it('lets a fresh document edge past a stale neighbour but not climb from the bottom', () => { + const rows = [ + row('stale-1', daysAgo(400)), + row('fresh-2', NOW), + ...Array.from({ length: 30 }, (_, i) => row(`mid-${i + 3}`, daysAgo(400))), + row('fresh-last', NOW), + ] + const ordered = applyRecencyBoost(rows, NOW).map((r) => r.id) + expect(ordered[0]).toBe('fresh-2') + expect(ordered[1]).toBe('stale-1') + expect(ordered.indexOf('fresh-last')).toBeGreaterThan(20) + }) + + it('does not mutate its input', () => { + const rows = [row('a', daysAgo(400)), row('b', NOW)] + applyRecencyBoost(rows, NOW) + expect(rows.map((r) => r.id)).toEqual(['a', 'b']) + }) +}) diff --git a/apps/sim/lib/knowledge/search/recency.ts b/apps/sim/lib/knowledge/search/recency.ts new file mode 100644 index 00000000000..d06dd12bacf --- /dev/null +++ b/apps/sim/lib/knowledge/search/recency.ts @@ -0,0 +1,46 @@ +/** + * Reciprocal-rank-fusion damping constant, shared by fusion and the recency + * boost so a rank means the same to both: `score = 1 / (RRF_K + rank)`. 60 is + * the value from the original RRF paper and matches the docs search retriever + * (`apps/docs/app/api/search/route.ts`). + */ +export const RRF_K = 60 + +/** Age at which a document's recency boost has decayed to half. */ +export const RECENCY_HALF_LIFE_DAYS = 90 +/** The most a fully fresh document's rank score is raised, as a fraction. */ +export const RECENCY_WEIGHT = 0.05 + +const DAY_MS = 24 * 60 * 60 * 1000 + +/** + * How fresh a document is on a 0..1 scale: 1 when modified now, halving every + * {@link RECENCY_HALF_LIFE_DAYS}. A document whose source reports no modified + * time, or one dated in the future, gets no boost at all. + */ +export function recencyFreshness(sourceModifiedAt: Date | null | undefined, now: Date): number { + if (!sourceModifiedAt) return 0 + const ageMs = now.getTime() - sourceModifiedAt.getTime() + if (!Number.isFinite(ageMs) || ageMs < 0) return 0 + return 2 ** (-ageMs / (RECENCY_HALF_LIFE_DAYS * DAY_MS)) +} + +/** + * Reorders relevance-ranked rows so a recently modified document edges past a + * stale one of similar relevance. The boost works on rank, not on the legs' + * incomparable raw scores, and it is bounded by {@link RECENCY_WEIGHT}: a + * fresh document can climb a handful of places, never from the bottom to the + * top. Rows without a source modified time keep their exact rank order. + */ +export function applyRecencyBoost( + rows: readonly T[], + now: Date = new Date() +): T[] { + const boosted = rows.map((row, index) => ({ + row, + score: + (1 / (RRF_K + index + 1)) * + (1 + RECENCY_WEIGHT * recencyFreshness(row.sourceModifiedAt, now)), + })) + return boosted.sort((a, b) => b.score - a.score).map((entry) => entry.row) +} diff --git a/apps/sim/lib/knowledge/search/snippet.test.ts b/apps/sim/lib/knowledge/search/snippet.test.ts new file mode 100644 index 00000000000..d468691e314 --- /dev/null +++ b/apps/sim/lib/knowledge/search/snippet.test.ts @@ -0,0 +1,105 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + findTermMatches, + matchSnippet, + queryTerms, + SNIPPET_LENGTH, + stripLeadingHeaders, +} from '@/lib/knowledge/search/snippet' + +const EMAIL = [ + 'Subject: Invoice #1010 is overdue', + 'From: Support ', + 'To: Someone ', + 'Messages: 1', + '', + `${'Thanks for your patience. '.repeat(12)}The Volvo order shipped on Monday and the tracking number follows. ${'More text here. '.repeat(20)}`, +].join('\n') + +const EVENT = ['Title: Weekly sync', 'Organizer: Ada', 'When: Monday 9am', 'Where: Room 4'].join( + '\n' +) + +describe('stripLeadingHeaders', () => { + it('drops the header block a connector writes above an email body', () => { + expect(stripLeadingHeaders(EMAIL).startsWith('\nThanks for your patience.')).toBe(true) + }) + + it('leaves a document that does not start with headers alone', () => { + expect(stripLeadingHeaders('Plain prose: with a colon inside.')).toBe( + 'Plain prose: with a colon inside.' + ) + }) + + it('keeps a chunk that is nothing but fields, such as a calendar event', () => { + expect(stripLeadingHeaders(EVENT)).toBe(EVENT) + expect(stripLeadingHeaders(`${EVENT}\n\n`)).toBe(`${EVENT}\n\n`) + }) +}) + +describe('queryTerms', () => { + it('keeps distinct terms of three or more characters, longest first', () => { + expect(queryTerms('the Volvo invoice is volvo')).toEqual(['invoice', 'Volvo', 'volvo', 'the']) + expect(queryTerms(undefined)).toEqual([]) + }) + + it('strips the quotes and punctuation around a term', () => { + expect(queryTerms('"foo bar" (baz),')).toEqual(['foo', 'bar', 'baz']) + }) +}) + +describe('findTermMatches', () => { + it('matches whole words in any script', () => { + expect(findTermMatches('Der Bericht über Zürich.', ['Zürich'])).toEqual([ + { index: 17, length: 6 }, + ]) + expect(findTermMatches('Reports on Zürichsee.', ['Zürich'])).toEqual([]) + expect(findTermMatches('東京の天気', ['天気'])).toEqual([{ index: 3, length: 2 }]) + }) + + it('reads whole characters beside a hit, not code units', () => { + expect(findTermMatches('𝔘nicode volvo𝔘 volvo', ['volvo'])).toEqual([{ index: 17, length: 5 }]) + }) + + it('skips a hit glued to another word character', () => { + expect(findTermMatches('subvolvo volvo_x volvo', ['volvo'])).toEqual([{ index: 17, length: 5 }]) + }) +}) + +describe('matchSnippet', () => { + it('returns a short document whole, without its headers', () => { + expect(matchSnippet('Subject: Hi\nFrom: A\n\nShort body.', 'body')).toBe('Short body.') + }) + + it('windows around the first query term with ellipses on both sides', () => { + const snippet = matchSnippet(EMAIL, 'volvo') + expect(snippet.startsWith('…')).toBe(true) + expect(snippet.endsWith('…')).toBe(true) + expect(snippet).toContain('The Volvo order shipped') + expect(snippet).not.toContain('Subject:') + expect(snippet.length).toBeLessThanOrEqual(SNIPPET_LENGTH + 2) + }) + + it('centres on a quoted phrase and on a non-ASCII term', () => { + expect(matchSnippet(EMAIL, '"Volvo order"')).toContain('The Volvo order shipped') + const german = `${'Einleitung. '.repeat(30)}Die Lieferung nach Zürich ist unterwegs. ${'Mehr. '.repeat(30)}` + expect(matchSnippet(german, 'Zürich')).toContain('nach Zürich') + }) + + it('never splits a surrogate pair at a window edge', () => { + const emoji = `${'🙂'.repeat(200)} volvo ${'🙂'.repeat(200)}` + const loneSurrogate = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? { + const snippet = matchSnippet(EMAIL, 'unrelated') + expect(snippet.startsWith('Thanks for your patience.')).toBe(true) + expect(snippet.endsWith('…')).toBe(true) + }) +}) diff --git a/apps/sim/lib/knowledge/search/snippet.ts b/apps/sim/lib/knowledge/search/snippet.ts new file mode 100644 index 00000000000..b1201b30bc0 --- /dev/null +++ b/apps/sim/lib/knowledge/search/snippet.ts @@ -0,0 +1,126 @@ +/** Characters of a document shown under a search result. */ +export const SNIPPET_LENGTH = 280 +/** Characters kept before the first match, so the hit sits in context rather than at the edge. */ +const LEAD_LENGTH = 90 +/** Query terms shorter than this are too common to anchor a snippet on. */ +const MIN_TERM_LENGTH = 3 +/** `Key: value` lines a connector writes above an email or ticket body. */ +const HEADER_LINE = /^[A-Z][A-Za-z-]{1,15}: .*$/ +/** + * A character that continues a word, so a term touching one on either side is + * part of a longer word rather than a hit. Scripts written without spaces + * (Han, kana, Hangul, Thai) have no such edges, so their letters never + * disqualify a neighbouring match. + */ +const WORD_CHARACTER = + /(?![\p{sc=Han}\p{sc=Hiragana}\p{sc=Katakana}\p{sc=Hangul}\p{sc=Thai}])[\p{L}\p{N}_]/u + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') +} + +/** + * The document text without the header block some connectors prefix (the + * `Subject:` / `From:` / `To:` lines of an email): the title already says + * what the subject is, and a snippet spent on the header never shows why the + * document matched. Only a block the connector closed with a blank line + * counts, and only when a body follows it: a chunk that is nothing but + * `Key: value` fields, such as a calendar event, is the document. + */ +export function stripLeadingHeaders(content: string): string { + const lines = content.split('\n') + let index = 0 + while (index < lines.length && HEADER_LINE.test(lines[index].trim())) index += 1 + if (index === 0 || index >= lines.length || lines[index].trim() !== '') return content + const body = lines.slice(index).join('\n') + return body.trim() ? body : content +} + +/** + * The query's terms worth matching, longest first so the most specific one + * wins: quotes and other search syntax around a term are not part of it. + */ +export function queryTerms(query: string | undefined): string[] { + return [ + ...new Set( + (query ?? '') + .split(/\s+/) + .map((term) => term.replace(/^["'“”‘’(]+|["'“”‘’),.;:!?]+$/g, '').trim()) + .filter((term) => term.length >= MIN_TERM_LENGTH) + ), + ].sort((a, b) => b.length - a.length) +} + +export interface TermMatch { + index: number + length: number +} + +/** + * Where the query terms occur in the text as whole words, in order and without + * overlap. Word edges are judged by the characters around a hit rather than + * by `\b`, which knows only ASCII letters, so a term in any script still + * matches; a hit glued to another word character on either side is not a + * word and is skipped. + */ +export function findTermMatches(text: string, terms: readonly string[]): TermMatch[] { + if (terms.length === 0) return [] + const pattern = new RegExp(terms.map(escapeRegExp).join('|'), 'giu') + const matches: TermMatch[] = [] + for (const match of text.matchAll(pattern)) { + const before = codePointBefore(text, match.index) + const after = codePointAt(text, match.index + match[0].length) + if (before !== undefined && WORD_CHARACTER.test(before)) continue + if (after !== undefined && WORD_CHARACTER.test(after)) continue + matches.push({ index: match.index, length: match[0].length }) + } + return matches +} + +/** The whole character starting at a code-unit index, or undefined past the end. */ +function codePointAt(text: string, index: number): string | undefined { + const code = text.codePointAt(index) + return code === undefined ? undefined : String.fromCodePoint(code) +} + +/** The whole character ending just before a code-unit index, or undefined at the start. */ +function codePointBefore(text: string, index: number): string | undefined { + if (index <= 0) return undefined + const unit = text.charCodeAt(index - 1) + const start = unit >= 0xdc00 && unit <= 0xdfff && index >= 2 ? index - 2 : index - 1 + return codePointAt(text, start) +} + +/** An index moved off the middle of a surrogate pair, so a slice never splits a character. */ +function alignToCodePoint(text: string, index: number): number { + const unit = text.charCodeAt(index) + return unit >= 0xdc00 && unit <= 0xdfff ? index - 1 : index +} + +/** + * The passage of a document a search result shows: a window around the first + * query term found, the way a search page shows why a document matched, and + * the document's opening when no term appears in this chunk. Whitespace is + * collapsed and the window is cut on word boundaries with ellipses where the + * text continues. + */ +export function matchSnippet(content: string, query?: string): string { + const flat = stripLeadingHeaders(content).replace(/\s+/g, ' ').trim() + if (flat.length <= SNIPPET_LENGTH) return flat + + const first = findTermMatches(flat, queryTerms(query))[0] + let start = first ? Math.max(0, first.index - LEAD_LENGTH) : 0 + if (start > 0) { + const boundary = flat.indexOf(' ', start) + if (boundary !== -1 && boundary - start < LEAD_LENGTH) start = boundary + 1 + } + start = alignToCodePoint(flat, start) + if (flat.length - start <= SNIPPET_LENGTH) { + return `${start > 0 ? '…' : ''}${flat.slice(start)}` + } + let end = start + SNIPPET_LENGTH + const lastSpace = flat.lastIndexOf(' ', end) + if (lastSpace > start + SNIPPET_LENGTH / 2) end = lastSpace + end = alignToCodePoint(flat, end) + return `${start > 0 ? '…' : ''}${flat.slice(start, end).trimEnd()}…` +} diff --git a/apps/sim/lib/knowledge/service.ts b/apps/sim/lib/knowledge/service.ts index 534bab008ce..0688c24755a 100644 --- a/apps/sim/lib/knowledge/service.ts +++ b/apps/sim/lib/knowledge/service.ts @@ -29,6 +29,7 @@ import { import { OrchestrationError } from '@/lib/core/orchestration/types' import { generateRestoreName } from '@/lib/core/utils/restore-name' import { findActiveFolder, resolveRestoredFolderId } from '@/lib/folders/queries' +import { isKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' import type { ChunkingConfig, CreateKnowledgeBaseData, @@ -177,7 +178,7 @@ async function readKnowledgeBaseRows( where: SQL | undefined, orderBy: SQL[], limit?: number -): Promise>> { +): Promise>> { const query = db .select({ id: knowledgeBase.id, @@ -219,7 +220,9 @@ async function readKnowledgeBaseRows( } async function attachConnectorTypes( - knowledgeBases: Array> + knowledgeBases: Array< + Omit + > ): Promise { const kbIds = knowledgeBases.map((kb) => kb.id) const connectorRows = @@ -228,6 +231,7 @@ async function attachConnectorTypes( .select({ knowledgeBaseId: knowledgeConnector.knowledgeBaseId, connectorType: knowledgeConnector.connectorType, + accessMode: knowledgeConnector.accessMode, }) .from(knowledgeConnector) .where( @@ -240,15 +244,33 @@ async function attachConnectorTypes( : [] const connectorTypesByKb = new Map() + const memberScopedKbIds = new Set() for (const row of connectorRows) { const types = connectorTypesByKb.get(row.knowledgeBaseId) ?? [] if (!types.includes(row.connectorType)) types.push(row.connectorType) connectorTypesByKb.set(row.knowledgeBaseId, types) + if (row.accessMode === 'members') memberScopedKbIds.add(row.knowledgeBaseId) + } + /** + * A members-mode connector only scopes documents where the feature is on; + * off, its documents read as workspace-visible and the base must say so. + */ + const memberScopedWorkspaceIds = new Set( + knowledgeBases + .filter((kb) => memberScopedKbIds.has(kb.id) && kb.workspaceId) + .map((kb) => kb.workspaceId as string) + ) + for (const workspaceId of memberScopedWorkspaceIds) { + if (await isKnowledgeMemberAccessAvailable({ workspaceId })) continue + for (const kb of knowledgeBases) { + if (kb.workspaceId === workspaceId) memberScopedKbIds.delete(kb.id) + } } return knowledgeBases.map((kb) => ({ ...kb, connectorTypes: connectorTypesByKb.get(kb.id) ?? [], + hasMemberScopedConnector: memberScopedKbIds.has(kb.id), })) } @@ -262,7 +284,7 @@ async function readWorkspaceKnowledgeBaseRows( scope: KnowledgeBaseScope, options?: GetKnowledgeBasesOptions ): Promise<{ - data: Array> + data: Array> nextCursorKeys: CursorKey[] | null }> { const { @@ -326,7 +348,7 @@ export async function getWorkspaceKnowledgeBases( async function readLegacyPersonalKnowledgeBaseRows( userId: string, scope: KnowledgeBaseScope -): Promise>> { +): Promise>> { const rows = await readKnowledgeBaseRows( and( knowledgeBaseScopeCondition(scope), @@ -382,7 +404,7 @@ export async function listWorkspaceAndLegacyKnowledgeBases( export async function findActiveKnowledgeBasesByExactName( workspaceId: string, name: string -): Promise>> { +): Promise>> { return readKnowledgeBaseRows( and( eq(knowledgeBase.workspaceId, workspaceId), @@ -485,6 +507,7 @@ export async function createAuthorizedKnowledgeBase( folderId, docCount: 0, connectorTypes: [], + hasMemberScopedConnector: false, } } @@ -923,12 +946,14 @@ export async function updateKnowledgeBase( logger.info(`[${requestId}] Updated knowledge base: ${knowledgeBaseId}`) - return { - ...updatedKb[0], - chunkingConfig: updatedKb[0].chunkingConfig as ChunkingConfig, - docCount: Number(updatedKb[0].docCount), - connectorTypes: [], - } + const [withConnectors] = await attachConnectorTypes([ + { + ...updatedKb[0], + chunkingConfig: updatedKb[0].chunkingConfig as ChunkingConfig, + docCount: Number(updatedKb[0].docCount), + }, + ]) + return withConnectors } /** @@ -1004,9 +1029,22 @@ export async function getKnowledgeBaseById( chunkingConfig: result[0].chunkingConfig as ChunkingConfig, docCount: Number(result[0].docCount), connectorTypes: [], + hasMemberScopedConnector: false, } } +/** + * The knowledge base with its connector summary, for the surfaces that show + * it. Kept off {@link getKnowledgeBaseById} so every operation that only + * resolves its context does not pay for the connector read. + */ +export async function attachKnowledgeBaseConnectors( + knowledgeBase: KnowledgeBaseWithCounts +): Promise { + const [withConnectors] = await attachConnectorTypes([knowledgeBase]) + return withConnectors +} + /** * Delete a knowledge base (soft delete) * diff --git a/apps/sim/lib/knowledge/tags/service.ts b/apps/sim/lib/knowledge/tags/service.ts index 5d94d736c57..6d1d112cf60 100644 --- a/apps/sim/lib/knowledge/tags/service.ts +++ b/apps/sim/lib/knowledge/tags/service.ts @@ -12,6 +12,8 @@ import { generateId } from '@sim/utils/id' import { and, eq, isNotNull, isNull, or, sql } from 'drizzle-orm' import { OrchestrationError } from '@/lib/core/orchestration/types' import type { DbOrTx, DbTransaction } from '@/lib/db/types' +import { knowledgeAccessCondition } from '@/lib/knowledge/access/predicate' +import type { KnowledgeAccessScope } from '@/lib/knowledge/access/types' import { getSlotsForFieldType, isValidSlotForFieldType, @@ -803,7 +805,8 @@ export async function updateTagDefinition( */ export async function getTagUsage( knowledgeBaseId: string, - requestId = 'api' + requestId: string, + access: KnowledgeAccessScope ): Promise< Array<{ tagName: string @@ -829,6 +832,7 @@ export async function getTagUsage( eq(document.userExcluded, false), isNull(document.archivedAt), isNull(document.deletedAt), + knowledgeAccessCondition(access), isNotNull(sql`${sql.raw(tagSlot)}`), ] @@ -867,6 +871,7 @@ export async function getTagUsage( */ export async function getTagUsageStats( knowledgeBaseId: string, + access: KnowledgeAccessScope, requestId: string ): Promise< Array<{ @@ -894,6 +899,7 @@ export async function getTagUsageStats( eq(document.userExcluded, false), isNull(document.archivedAt), isNull(document.deletedAt), + knowledgeAccessCondition(access), sql`${sql.raw(tagSlot)} IS NOT NULL` ) ) @@ -908,6 +914,7 @@ export async function getTagUsageStats( eq(document.userExcluded, false), isNull(document.archivedAt), isNull(document.deletedAt), + knowledgeAccessCondition(access), sql`${sql.raw(`embedding.${tagSlot}`)} IS NOT NULL` ) ) diff --git a/apps/sim/lib/knowledge/types.ts b/apps/sim/lib/knowledge/types.ts index 3444945ef41..7a7d22666a4 100644 --- a/apps/sim/lib/knowledge/types.ts +++ b/apps/sim/lib/knowledge/types.ts @@ -30,6 +30,8 @@ export interface KnowledgeBaseWithCounts { folderId: string | null docCount: number connectorTypes: string[] + /** True when a live connector syncs per member, so what a run retrieves depends on who triggers it. */ + hasMemberScopedConnector: boolean } export interface CreateKnowledgeBaseData { @@ -121,6 +123,7 @@ export interface KnowledgeBaseData { folderId: string | null docCount?: number connectorTypes?: string[] + hasMemberScopedConnector?: boolean } export interface DocumentData { @@ -204,3 +207,11 @@ interface DocumentsPagination { offset: number hasMore: boolean } + +/** The member engine's states, as stored on `knowledge_connector.member_sync_status`. */ +export const MEMBER_SYNC_STATUSES = ['idle', 'pending', 'running', 'error', 'disabled'] as const +export type MemberSyncStatus = (typeof MEMBER_SYNC_STATUSES)[number] + +export function isMemberSyncStatus(value: string): value is MemberSyncStatus { + return (MEMBER_SYNC_STATUSES as readonly string[]).includes(value) +} diff --git a/apps/sim/lib/mothership/events.ts b/apps/sim/lib/mothership/events.ts index 4ce4d844da8..efb80e7eaf4 100644 --- a/apps/sim/lib/mothership/events.ts +++ b/apps/sim/lib/mothership/events.ts @@ -1,5 +1,8 @@ import { createLogger } from '@sim/logger' -import type { FileAttachmentForApi } from '@/app/workspace/[workspaceId]/home/types' +import type { + ChatRequestMode, + FileAttachmentForApi, +} from '@/app/workspace/[workspaceId]/home/types' import type { ChatContext } from '@/stores/panel' const logger = createLogger('MothershipEvents') @@ -34,6 +37,8 @@ export interface MothershipSendMessageDetail { * chat and billing a second turn. */ resumeUserMessageId?: string + /** The request mode the withdrawn send asked for, so a retry stays the same kind of turn. */ + requestMode?: ChatRequestMode } /** @@ -49,7 +54,8 @@ export function sendMothershipMessage( message: string, contexts?: ChatContext[], fileAttachments?: FileAttachmentForApi[], - resumeUserMessageId?: string + resumeUserMessageId?: string, + requestMode?: ChatRequestMode ): boolean { const trimmed = message.trim() if (!trimmed) { @@ -61,6 +67,7 @@ export function sendMothershipMessage( contexts, fileAttachments, ...(resumeUserMessageId ? { resumeUserMessageId } : {}), + ...(requestMode ? { requestMode } : {}), }) logger.info('Dispatched mothership message event', { messageLength: trimmed.length, consumed }) return consumed diff --git a/apps/sim/lib/oauth/token-resolution.ts b/apps/sim/lib/oauth/token-resolution.ts index ba903ef7eb5..dfd1f682b5b 100644 --- a/apps/sim/lib/oauth/token-resolution.ts +++ b/apps/sim/lib/oauth/token-resolution.ts @@ -1,8 +1,5 @@ import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' -import { - resolvePrincipalSubject, - type WorkflowExecutionDelegatedPrincipal, -} from '@sim/auth/principal' +import { type DelegatedPrincipal, resolvePrincipalSubject } from '@sim/auth/principal' import { createLogger } from '@sim/logger' import { impersonateEmailSchema, @@ -326,13 +323,13 @@ export interface ResolveCredentialAccessTokenInput */ authenticate: () => AuthResult | Promise /** - * Proves a workflow-execution delegation for one managed credential. The route - * verifies the delegation JWT header; the executor binds its delegation origin - * in-process. Absent, managed credentials are rejected with - * `MANAGED_CREDENTIAL_DELEGATION_REQUIRED`. Must throw - * {@link InvalidManagedOAuthDelegationError} on an invalid delegation. + * Proves a delegation for one managed credential: a workflow execution (the + * route verifies the delegation JWT header; the executor binds its delegation + * origin in-process) or a Chat turn acting as the signed-in user. Absent, + * managed credentials are rejected with `MANAGED_CREDENTIAL_DELEGATION_REQUIRED`. + * Must throw {@link InvalidManagedOAuthDelegationError} on an invalid delegation. */ - resolveManagedPrincipal?: (credentialId: string) => Promise + resolveManagedPrincipal?: (credentialId: string) => Promise } /** @@ -376,7 +373,7 @@ export async function resolveCredentialAccessToken( } } - let principal: WorkflowExecutionDelegatedPrincipal + let principal: DelegatedPrincipal try { principal = await input.resolveManagedPrincipal(resolved.credentialId) } catch (error) { diff --git a/apps/sim/lib/posthog/events.ts b/apps/sim/lib/posthog/events.ts index 85da54723a7..982f963fa02 100644 --- a/apps/sim/lib/posthog/events.ts +++ b/apps/sim/lib/posthog/events.ts @@ -614,10 +614,19 @@ export interface PostHogEventMap { action_id?: string } - /** A home-page suggested action was clicked. `action_id` is the candidate id (e.g. `gmail-0`). */ + /** The chat composer's mode switcher picked a different mode. */ + chat_mode_changed: { + workspace_id: string + mode: 'build' | 'search' | 'assistant' + } + + /** + * A home-page suggested action was clicked. `action_id` is the candidate id + * (e.g. `gmail-0`); `connector` rows are the Search-mode "Connect X" rows. + */ suggested_action_clicked: { workspace_id: string - kind: 'prompt' | 'integration' + kind: 'prompt' | 'integration' | 'connector' action_id: string label: string position: number diff --git a/apps/sim/lib/resource-policies/conditions/credential-group-option.ts b/apps/sim/lib/resource-policies/conditions/credential-group-option.ts new file mode 100644 index 00000000000..96a5d8315c6 --- /dev/null +++ b/apps/sim/lib/resource-policies/conditions/credential-group-option.ts @@ -0,0 +1,18 @@ +import { defineResourcePolicyCondition } from '@/lib/resource-policies/conditions/types' + +export const CREDENTIAL_GROUP_OPTION_ID_CONDITION_KEY = 'credential_group:OptionId' as const + +/** + * The Credential Group option the credential being accessed was collected + * under. A statement conditioned on it grants one option's credentials and no + * other, which is how a knowledge connector is bound to exactly the provider + * slot it crawls with. + */ +export const credentialGroupOptionIdConditionDefinition = defineResourcePolicyCondition({ + key: CREDENTIAL_GROUP_OPTION_ID_CONDITION_KEY, + label: 'Credential option', + valueType: 'string', + operators: ['StringEquals'], + selector: { type: 'internal' }, + resolve: (facts) => facts.credentialGroupOptionId, +}) diff --git a/apps/sim/lib/resource-policies/conditions/index.ts b/apps/sim/lib/resource-policies/conditions/index.ts index 6796a069ac6..dc65677caac 100644 --- a/apps/sim/lib/resource-policies/conditions/index.ts +++ b/apps/sim/lib/resource-policies/conditions/index.ts @@ -1,4 +1,5 @@ export { CREDENTIAL_GROUP_ACTOR_OWNS_CREDENTIAL_CONDITION_KEY } from '@/lib/resource-policies/conditions/credential-group-actor-owns-credential' +export { CREDENTIAL_GROUP_OPTION_ID_CONDITION_KEY } from '@/lib/resource-policies/conditions/credential-group-option' export { getResourcePolicyConditionDefinition, RESOURCE_POLICY_CONDITION_DEFINITIONS, diff --git a/apps/sim/lib/resource-policies/conditions/registry.test.ts b/apps/sim/lib/resource-policies/conditions/registry.test.ts index b4aa346416e..65c70b8d8cc 100644 --- a/apps/sim/lib/resource-policies/conditions/registry.test.ts +++ b/apps/sim/lib/resource-policies/conditions/registry.test.ts @@ -48,6 +48,15 @@ describe('resource policy condition registry', () => { ) }) + it('registers the credential option as an internal string fact', () => { + const definition = RESOURCE_POLICY_CONDITION_DEFINITIONS['credential_group:OptionId'] + expect(definition.valueType).toBe('string') + expect(definition.operators).toEqual(['StringEquals']) + expect(definition.selector).toEqual({ type: 'internal' }) + expect(definition.resolve({ credentialGroupOptionId: 'option-1' })).toBe('option-1') + expect(definition.resolve({})).toBe(undefined) + }) + it('fails fast for an unregistered condition key', () => { expect(() => requireResourcePolicyConditionDefinition('execution:Unknown')).toThrow( 'Resource policy condition key execution:Unknown is not registered' diff --git a/apps/sim/lib/resource-policies/conditions/registry.ts b/apps/sim/lib/resource-policies/conditions/registry.ts index 454ede858a3..780dfded096 100644 --- a/apps/sim/lib/resource-policies/conditions/registry.ts +++ b/apps/sim/lib/resource-policies/conditions/registry.ts @@ -1,4 +1,5 @@ import { credentialGroupActorOwnsCredentialConditionDefinition } from '@/lib/resource-policies/conditions/credential-group-actor-owns-credential' +import { credentialGroupOptionIdConditionDefinition } from '@/lib/resource-policies/conditions/credential-group-option' import type { ResourcePolicyConditionDefinition, ResourcePolicyConditionKey, @@ -7,6 +8,7 @@ import { workflowModeResourcePolicyConditionDefinition } from '@/lib/resource-po export const RESOURCE_POLICY_CONDITION_DEFINITIONS = Object.freeze({ 'credential_group:ActorOwnsCredential': credentialGroupActorOwnsCredentialConditionDefinition, + 'credential_group:OptionId': credentialGroupOptionIdConditionDefinition, 'execution:WorkflowMode': workflowModeResourcePolicyConditionDefinition, } as const satisfies Record) diff --git a/apps/sim/lib/resource-policies/conditions/types.ts b/apps/sim/lib/resource-policies/conditions/types.ts index d0e834177c0..d22c9397071 100644 --- a/apps/sim/lib/resource-policies/conditions/types.ts +++ b/apps/sim/lib/resource-policies/conditions/types.ts @@ -5,6 +5,8 @@ export type ResourcePolicyConditionOperator = (typeof RESOURCE_POLICY_CONDITION_ export interface ResourcePolicyConditionEvaluationFacts { credentialGroupActorEnrollmentId?: string credentialGroupCredentialEnrollmentId?: string + /** The option the credential being accessed was collected under. */ + credentialGroupOptionId?: string currentWorkflow?: { workflowId: string mode: 'draft' | 'deployment' @@ -33,6 +35,7 @@ export interface ResourcePolicyConditionDefinition { export type ResourcePolicyConditionKey = | 'credential_group:ActorOwnsCredential' + | 'credential_group:OptionId' | 'execution:WorkflowMode' export function defineResourcePolicyCondition( diff --git a/apps/sim/lib/resource-policies/principals/index.ts b/apps/sim/lib/resource-policies/principals/index.ts index 303918f1513..a967047f431 100644 --- a/apps/sim/lib/resource-policies/principals/index.ts +++ b/apps/sim/lib/resource-policies/principals/index.ts @@ -1,4 +1,5 @@ export { credentialGroupActorResourcePolicyPrincipalSchema } from '@/lib/resource-policies/principals/credential-group-actor' +export { knowledgeConnectorResourcePolicyPrincipalSchema } from '@/lib/resource-policies/principals/knowledge-connector' export { getResourcePolicyPrincipalDefinition, matchResourcePolicyPrincipal, @@ -7,6 +8,7 @@ export { } from '@/lib/resource-policies/principals/registry' export type { CredentialGroupActorResourcePolicyPrincipal, + KnowledgeConnectorResourcePolicyPrincipal, ResourcePolicyPrincipal, ResourcePolicyPrincipalDefinition, ResourcePolicyPrincipalEvaluationFacts, diff --git a/apps/sim/lib/resource-policies/principals/knowledge-connector.ts b/apps/sim/lib/resource-policies/principals/knowledge-connector.ts new file mode 100644 index 00000000000..3f26fac46a1 --- /dev/null +++ b/apps/sim/lib/resource-policies/principals/knowledge-connector.ts @@ -0,0 +1,27 @@ +import { z } from 'zod' +import { defineResourcePolicyPrincipal } from '@/lib/resource-policies/principals/types' + +export const knowledgeConnectorResourcePolicyPrincipalSchema = z + .object({ + type: z.literal('knowledge_connector'), + connectorId: z + .string() + .min(1) + .max(128) + .refine((value) => value === value.trim(), 'Knowledge connector ID must be canonical'), + }) + .strict() + +/** + * A knowledge connector crawling a source on behalf of the people enrolled in a + * Credential Group. Granted by the connector's own settings, never from the + * group's access page, so the selector is internal. + */ +export const knowledgeConnectorResourcePolicyPrincipalDefinition = defineResourcePolicyPrincipal({ + type: 'knowledge_connector', + schema: knowledgeConnectorResourcePolicyPrincipalSchema, + label: 'Knowledge connector', + selector: { type: 'internal' }, + matches: (principal, facts) => + facts.currentKnowledgeConnector?.connectorId === principal.connectorId, +}) diff --git a/apps/sim/lib/resource-policies/principals/registry.test.ts b/apps/sim/lib/resource-policies/principals/registry.test.ts index bb98a8de73f..36a1f6ed35b 100644 --- a/apps/sim/lib/resource-policies/principals/registry.test.ts +++ b/apps/sim/lib/resource-policies/principals/registry.test.ts @@ -42,6 +42,27 @@ describe('resource policy principal registry', () => { ).toBe(false) }) + it('registers the internal knowledge connector principal', () => { + expect(RESOURCE_POLICY_PRINCIPAL_DEFINITIONS.knowledge_connector.selector).toEqual({ + type: 'internal', + }) + expect( + matchResourcePolicyPrincipal( + { type: 'knowledge_connector', connectorId: 'connector-1' }, + { currentKnowledgeConnector: { connectorId: 'connector-1' } } + ) + ).toBe(true) + expect( + matchResourcePolicyPrincipal( + { type: 'knowledge_connector', connectorId: 'connector-1' }, + { currentKnowledgeConnector: { connectorId: 'connector-2' } } + ) + ).toBe(false) + expect( + matchResourcePolicyPrincipal({ type: 'knowledge_connector', connectorId: 'connector-1' }, {}) + ).toBe(false) + }) + it('fails fast for an unregistered principal type', () => { expect(() => requireResourcePolicyPrincipalDefinition('user')).toThrow( 'Resource policy principal type user is not registered' diff --git a/apps/sim/lib/resource-policies/principals/registry.ts b/apps/sim/lib/resource-policies/principals/registry.ts index 9649267f5f8..923490c8bf1 100644 --- a/apps/sim/lib/resource-policies/principals/registry.ts +++ b/apps/sim/lib/resource-policies/principals/registry.ts @@ -1,4 +1,5 @@ import { credentialGroupActorResourcePolicyPrincipalDefinition } from '@/lib/resource-policies/principals/credential-group-actor' +import { knowledgeConnectorResourcePolicyPrincipalDefinition } from '@/lib/resource-policies/principals/knowledge-connector' import type { ResourcePolicyPrincipal, ResourcePolicyPrincipalDefinition, @@ -9,6 +10,7 @@ import { workflowResourcePolicyPrincipalDefinition } from '@/lib/resource-polici export const RESOURCE_POLICY_PRINCIPAL_DEFINITIONS = Object.freeze({ credential_group_actor: credentialGroupActorResourcePolicyPrincipalDefinition, + knowledge_connector: knowledgeConnectorResourcePolicyPrincipalDefinition, workflow: workflowResourcePolicyPrincipalDefinition, } as const satisfies Record) diff --git a/apps/sim/lib/resource-policies/principals/types.ts b/apps/sim/lib/resource-policies/principals/types.ts index 1df07d52033..eb68f056736 100644 --- a/apps/sim/lib/resource-policies/principals/types.ts +++ b/apps/sim/lib/resource-policies/principals/types.ts @@ -9,9 +9,15 @@ export interface CredentialGroupActorResourcePolicyPrincipal { type: 'credential_group_actor' } +export interface KnowledgeConnectorResourcePolicyPrincipal { + type: 'knowledge_connector' + connectorId: string +} + export type ResourcePolicyPrincipal = | WorkflowResourcePolicyPrincipal | CredentialGroupActorResourcePolicyPrincipal + | KnowledgeConnectorResourcePolicyPrincipal export type ResourcePolicyPrincipalType = ResourcePolicyPrincipal['type'] export interface ResourcePolicyPrincipalEvaluationFacts { @@ -20,6 +26,9 @@ export interface ResourcePolicyPrincipalEvaluationFacts { workflowId: string mode: 'draft' | 'deployment' } + currentKnowledgeConnector?: { + connectorId: string + } } export type ResourcePolicyPrincipalSelector = diff --git a/apps/sim/lib/resource-policies/registry.ts b/apps/sim/lib/resource-policies/registry.ts index c59bee2fd1a..2ab5f068614 100644 --- a/apps/sim/lib/resource-policies/registry.ts +++ b/apps/sim/lib/resource-policies/registry.ts @@ -17,8 +17,12 @@ interface ResourcePolicyResourceDefinition { export const RESOURCE_POLICY_DEFINITIONS = Object.freeze({ credential_group: { actions: RESOURCE_POLICY_ACTIONS, - principalTypes: ['credential_group_actor', 'workflow'], - conditionKeys: ['credential_group:ActorOwnsCredential', 'execution:WorkflowMode'], + principalTypes: ['credential_group_actor', 'knowledge_connector', 'workflow'], + conditionKeys: [ + 'credential_group:ActorOwnsCredential', + 'credential_group:OptionId', + 'execution:WorkflowMode', + ], }, } as const satisfies Record) diff --git a/apps/sim/lib/sim-search/connectors.test.ts b/apps/sim/lib/sim-search/connectors.test.ts new file mode 100644 index 00000000000..0aec5f4aa92 --- /dev/null +++ b/apps/sim/lib/sim-search/connectors.test.ts @@ -0,0 +1,158 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('@/connectors/registry', () => { + const icon = () => null + return { + CONNECTOR_META_REGISTRY: { + mintlify: { id: 'mintlify', name: 'Mintlify', auth: { mode: 'apiKey' }, icon }, + jsm: { + id: 'jsm', + name: 'Jira Service Management', + auth: { mode: 'oauth', provider: 'jira' }, + configFields: [], + icon, + }, + jira: { + id: 'jira', + name: 'Jira', + auth: { mode: 'oauth', provider: 'jira' }, + permissionScopedListing: { capFieldIds: ['maxIssues'] }, + configFields: [{ id: 'domain', required: true }], + icon, + }, + google_drive: { + id: 'google_drive', + name: 'Google Drive', + auth: { mode: 'oauth', provider: 'google-drive' }, + permissionScopedListing: { capFieldIds: ['maxFiles'] }, + configFields: [{ id: 'maxFiles', required: false }], + icon, + }, + gmail: { + id: 'gmail', + name: 'Gmail', + auth: { mode: 'oauth', provider: 'google-email' }, + configFields: [], + icon, + }, + unknown: { + id: 'unknown', + name: 'Unknown', + auth: { mode: 'oauth', provider: 'not-a-service' }, + configFields: [], + icon, + }, + salesforce: { + id: 'salesforce', + name: 'Salesforce', + auth: { mode: 'oauth', provider: 'salesforce' }, + configFields: [], + icon, + }, + }, + } +}) + +vi.mock('@/lib/oauth', () => { + const services = { + jira: { providerId: 'jira', name: 'Jira', icon: () => null }, + 'google-drive': { providerId: 'google-drive', name: 'Google Drive', icon: () => null }, + gmail: { providerId: 'google-email', name: 'Gmail', icon: () => null }, + salesforce: { + providerId: 'salesforce', + name: 'Salesforce', + icon: () => null, + additionalProviderIds: ['salesforce-sandbox'], + }, + } + return { + getServiceConfigByServiceId: (serviceId: string) => + services[serviceId as keyof typeof services] ?? null, + getServiceConfigByProviderId: (providerId: string) => + Object.values(services).find((service) => service.providerId === providerId) ?? null, + getCanonicalScopesForProvider: (providerId: string) => [`${providerId}:read`], + } +}) + +vi.mock('@/lib/integrations/credential-display', () => ({ + getIntegrationsForCredentialProvider: (providerId: string) => + providerId === 'jira' ? [{ type: 'jira' }] : [], +})) + +import { + canConnectPersonally, + isSearchConnectorAvailable, + missingSetupFields, + personalSetupFields, + SEARCH_CONNECTORS, +} from '@/lib/sim-search/connectors' + +describe('SEARCH_CONNECTORS', () => { + it('lists OAuth connectors with a registered service, alphabetically', () => { + expect(SEARCH_CONNECTORS.map((connector) => connector.type)).toEqual([ + 'gmail', + 'google_drive', + 'jira', + 'jsm', + 'salesforce', + ]) + }) + + it('resolves the provider, scopes, and brand block type per connector', () => { + const jsm = SEARCH_CONNECTORS.find((connector) => connector.type === 'jsm') + expect(jsm).toMatchObject({ + providerId: 'jira', + providerIds: ['jira'], + requiredScopes: ['jira:read'], + serviceName: 'Jira', + blockType: 'jira', + }) + const drive = SEARCH_CONNECTORS.find((connector) => connector.type === 'google_drive') + expect(drive).toMatchObject({ blockType: 'google_drive' }) + const gmail = SEARCH_CONNECTORS.find((connector) => connector.type === 'gmail') + expect(gmail).toMatchObject({ providerId: 'google-email', serviceName: 'Gmail' }) + }) +}) + +describe('canConnectPersonally', () => { + it('offers personal connection to OAuth sources whose listing is permission-scoped', () => { + const drive = SEARCH_CONNECTORS.find((connector) => connector.type === 'google_drive')! + const jira = SEARCH_CONNECTORS.find((connector) => connector.type === 'jira')! + const gmail = SEARCH_CONNECTORS.find((connector) => connector.type === 'gmail')! + expect(canConnectPersonally(drive.meta)).toBe(true) + expect(canConnectPersonally(jira.meta)).toBe(true) + expect(canConnectPersonally(gmail.meta)).toBe(false) + }) +}) + +describe('personalSetupFields', () => { + it('asks for required config beyond the listing caps, never a selector', () => { + const drive = SEARCH_CONNECTORS.find((connector) => connector.type === 'google_drive')! + const jira = SEARCH_CONNECTORS.find((connector) => connector.type === 'jira')! + expect(personalSetupFields(drive.meta)).toEqual([]) + expect(personalSetupFields(jira.meta).map((field) => field.id)).toEqual(['domain']) + }) + + it('reports the setup fields a config leaves empty', () => { + const jira = SEARCH_CONNECTORS.find((connector) => connector.type === 'jira')! + expect(missingSetupFields(jira.meta, {}).map((field) => field.id)).toEqual(['domain']) + expect(missingSetupFields(jira.meta, { domain: ' ' })).toHaveLength(1) + expect(missingSetupFields(jira.meta, { domain: 'acme.atlassian.net' })).toEqual([]) + }) +}) + +describe('isSearchConnectorAvailable', () => { + it('reads the OAuth path of the connector’s block, defaulting to available', () => { + const jira = SEARCH_CONNECTORS.find((connector) => connector.type === 'jira')! + expect(isSearchConnectorAvailable(jira, new Map([['jira', { oauthAvailable: false }]]))).toBe( + false + ) + expect(isSearchConnectorAvailable(jira, new Map([['jira', { oauthAvailable: true }]]))).toBe( + true + ) + expect(isSearchConnectorAvailable(jira, new Map())).toBe(true) + }) +}) diff --git a/apps/sim/lib/sim-search/connectors.ts b/apps/sim/lib/sim-search/connectors.ts new file mode 100644 index 00000000000..00ce73c85ff --- /dev/null +++ b/apps/sim/lib/sim-search/connectors.ts @@ -0,0 +1,153 @@ +import type { ComponentType } from 'react' +import { getIntegrationsForCredentialProvider } from '@/lib/integrations/credential-display' +import { + getCanonicalScopesForProvider, + getServiceConfigByProviderId, + getServiceConfigByServiceId, +} from '@/lib/oauth' +import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' +import type { ConnectorConfigField, ConnectorMeta } from '@/connectors/types' + +/** The workspace knowledge base Sim Search indexes into, one per workspace, created on first connect. */ +export const SIM_SEARCH_KNOWLEDGE_BASE_NAME = 'Sim Search' + +/** + * A knowledge-base connector offered on the Sim Search surface: the connector's + * client-safe meta paired with the OAuth service a user connects it through. + * Only OAuth connectors qualify — an API-key connector has nowhere to keep a + * personal key outside a knowledge base, so it stays a knowledge-base flow. + */ +export interface SearchConnector { + /** `CONNECTOR_META_REGISTRY` key — the id a knowledge base reports in `connectorTypes`. */ + type: string + meta: ConnectorMeta + /** Canonical OAuth provider id the connection is stored under. */ + providerId: string + /** + * Every provider id a credential for this service may carry: the canonical + * id plus any additional authorization server (Salesforce sandbox). A + * credential under any of them counts as connected. + */ + providerIds: readonly string[] + /** + * Scopes listed in the connect modal — the provider's canonical set, which is + * what the knowledge-base connector flow requests for the same provider. + */ + requiredScopes: readonly string[] + /** The OAuth service's own name and mark, for the connect modal. */ + serviceName: string + serviceIcon: ComponentType<{ className?: string }> + /** + * Block type lending the brand tile and the deployment-availability lookup: + * the first catalog integration on the provider, else the connector type. + */ + blockType: string + /** Required config a person supplies on the source's first connect; empty for one-click sources. */ + setupFields: readonly ConnectorConfigField[] +} + +/** + * Every Sim Search connector, alphabetical by name. Built once at module load. + * + * A connector names its service by service id (`confluence`) or, for Gmail, by + * the provider id (`google-email`); the knowledge-base connector flow accepts + * both through `getProviderIdFromServiceId`'s raw fallback, so the lookup here + * tries the service id first and the provider id second. + */ +export const SEARCH_CONNECTORS: readonly SearchConnector[] = Object.entries(CONNECTOR_META_REGISTRY) + .flatMap(([type, meta]): SearchConnector[] => { + if (meta.auth.mode !== 'oauth') return [] + const service = + getServiceConfigByServiceId(meta.auth.provider) ?? + getServiceConfigByProviderId(meta.auth.provider) + if (!service) return [] + return [ + { + type, + meta, + providerId: service.providerId, + providerIds: [service.providerId, ...(service.additionalProviderIds ?? [])], + requiredScopes: getCanonicalScopesForProvider(service.providerId), + serviceName: service.name, + serviceIcon: service.icon as ComponentType<{ className?: string }>, + blockType: getIntegrationsForCredentialProvider(service.providerId)[0]?.type ?? type, + setupFields: personalSetupFields(meta), + }, + ] + }) + .sort((a, b) => a.meta.name.localeCompare(b.meta.name)) + +/** + * Whether a source connects per person on Sim Search: it authenticates with + * OAuth and its listing reflects who may read each document, so each member's + * own crawl is the permission check. A source that fails this is a workspace + * connector an admin sets up from a knowledge base. + */ +export function canConnectPersonally(meta: ConnectorMeta): boolean { + return meta.auth.mode === 'oauth' && meta.permissionScopedListing !== undefined +} + +/** + * The fields a person fills in before a source's first connect: its required + * config beyond the listing caps members mode clears. A selector needs a + * credential the source does not have yet, so a selector's typed twin stands + * in for it (Confluence's space key, Jira's project key). + */ +export function personalSetupFields(meta: ConnectorMeta): ConnectorConfigField[] { + const capFieldIds = new Set(meta.permissionScopedListing?.capFieldIds ?? []) + return meta.configFields.filter( + (field) => field.required && field.type !== 'selector' && !capFieldIds.has(field.id) + ) +} + +/** The setup fields a source config leaves empty. */ +export function missingSetupFields( + meta: ConnectorMeta, + sourceConfig: Record +): ConnectorConfigField[] { + return personalSetupFields(meta).filter((field) => !sourceConfig[field.id]?.trim()) +} + +/** The name a connector shows, from its registry entry. */ +export function connectorDisplayName(connectorType: string): string { + return CONNECTOR_META_REGISTRY[connectorType]?.name ?? connectorType +} + +export interface SearchConnectorAvailabilityContext { + /** Whether per-member access is on for the workspace. */ + memberAccessAvailable: boolean + /** Whether someone already connected this source in the workspace. */ + hasConnection: boolean + /** Whether the viewer may turn a source on for the workspace; the first connect needs an admin. */ + canCreate: boolean +} + +/** Why a source cannot be connected on this surface right now; null when it can. */ +export function searchConnectorUnavailableReason( + connector: SearchConnector, + integrationAvailability: ReadonlyMap, + context: SearchConnectorAvailabilityContext +): string | null { + if (!isSearchConnectorAvailable(connector, integrationAvailability)) { + return `${connector.meta.name} is unavailable in this deployment` + } + if (!context.memberAccessAvailable) return 'Per-member access is not available in this workspace' + if (!context.hasConnection && !context.canCreate) { + return `Ask a workspace admin to connect ${connector.meta.name} first` + } + return null +} + +/** + * Whether this deployment can connect the connector. The OAuth path + * specifically: an integration's `state` can read `limited` on a + * service-account-only deployment, but a connector authenticates with OAuth + * alone. A connector with no availability entry is assumed connectable. + */ +export function isSearchConnectorAvailable( + connector: SearchConnector, + integrationAvailability: ReadonlyMap +): boolean { + const availability = integrationAvailability.get(connector.blockType.toLowerCase()) + return availability ? availability.oauthAvailable : true +} diff --git a/apps/sim/lib/sim-search/knowledge-bases.test.ts b/apps/sim/lib/sim-search/knowledge-bases.test.ts new file mode 100644 index 00000000000..4229901af36 --- /dev/null +++ b/apps/sim/lib/sim-search/knowledge-bases.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from 'vitest' +import { + MAX_SEARCHED_KNOWLEDGE_BASES, + searchedKnowledgeBases, + withSearchedKnowledgeContexts, +} from '@/lib/sim-search/knowledge-bases' + +const base = (id: string, workspaceId: string | null = 'ws-1') => ({ + id, + name: `Base ${id}`, + workspaceId, +}) + +describe('searchedKnowledgeBases', () => { + it('keeps only the bases of the named workspace, capped', () => { + const bases = [ + base('legacy', null), + base('other', 'ws-2'), + ...Array.from({ length: MAX_SEARCHED_KNOWLEDGE_BASES + 1 }, (_, i) => base(`kb-${i}`)), + ] + const searched = searchedKnowledgeBases(bases, 'ws-1') + expect(searched).toHaveLength(MAX_SEARCHED_KNOWLEDGE_BASES) + expect(searched.every((kb) => kb.workspaceId === 'ws-1')).toBe(true) + }) +}) + +describe('withSearchedKnowledgeContexts', () => { + it('attaches every searched base after the contexts the person chose', () => { + expect( + withSearchedKnowledgeContexts( + [{ kind: 'file', fileId: 'f-1', label: 'notes.md' }], + [base('kb-1')] + ) + ).toEqual([ + { kind: 'file', fileId: 'f-1', label: 'notes.md' }, + { kind: 'knowledge', knowledgeId: 'kb-1', label: 'Base kb-1' }, + ]) + }) + + it('does not attach a base the person already mentioned', () => { + const mentioned = { kind: 'knowledge' as const, knowledgeId: 'kb-1', label: 'Mine' } + expect(withSearchedKnowledgeContexts([mentioned], [base('kb-1'), base('kb-2')])).toEqual([ + mentioned, + { kind: 'knowledge', knowledgeId: 'kb-2', label: 'Base kb-2' }, + ]) + }) + + it('returns the bases alone when nothing was chosen', () => { + expect(withSearchedKnowledgeContexts(undefined, [base('kb-1')])).toEqual([ + { kind: 'knowledge', knowledgeId: 'kb-1', label: 'Base kb-1' }, + ]) + }) +}) diff --git a/apps/sim/lib/sim-search/knowledge-bases.ts b/apps/sim/lib/sim-search/knowledge-bases.ts new file mode 100644 index 00000000000..b07727522ca --- /dev/null +++ b/apps/sim/lib/sim-search/knowledge-bases.ts @@ -0,0 +1,38 @@ +import type { KnowledgeBaseData } from '@/lib/api/contracts/knowledge/base' +import type { ChatContext } from '@/stores/panel/types' + +/** A search, or an answer drawn from one, spans at most this many knowledge bases. */ +export const MAX_SEARCHED_KNOWLEDGE_BASES = 20 + +type SearchedKnowledgeBase = Pick + +/** + * The bases a workspace search covers. The list also carries the viewer's + * legacy personal bases, which have no workspace; a search names one workspace + * and refuses a base outside it. + */ +export function searchedKnowledgeBases( + bases: readonly T[], + workspaceId: string +): T[] { + return bases.filter((kb) => kb.workspaceId === workspaceId).slice(0, MAX_SEARCHED_KNOWLEDGE_BASES) +} + +/** + * The contexts an Ask turn carries: every searched base, attached the way an + * `@` mention attaches one, so the agent answers from the same documents the + * Search panel shows. A base the person already mentioned is not attached twice. + */ +export function withSearchedKnowledgeContexts( + contexts: readonly ChatContext[] | undefined, + bases: readonly SearchedKnowledgeBase[] +): ChatContext[] { + const mentioned = new Set() + for (const context of contexts ?? []) { + if (context.kind === 'knowledge' && context.knowledgeId) mentioned.add(context.knowledgeId) + } + const attached: ChatContext[] = bases + .filter((kb) => !mentioned.has(kb.id)) + .map((kb) => ({ kind: 'knowledge', knowledgeId: kb.id, label: kb.name })) + return [...(contexts ?? []), ...attached] +} diff --git a/apps/sim/lib/uploads/utils/file-utils.server.ts b/apps/sim/lib/uploads/utils/file-utils.server.ts index d3cb23db6c9..b78bff2b00b 100644 --- a/apps/sim/lib/uploads/utils/file-utils.server.ts +++ b/apps/sim/lib/uploads/utils/file-utils.server.ts @@ -37,7 +37,7 @@ import { } from '@/lib/uploads/utils/file-utils' import { isSimPageSource, SIM_PAGE_CONTENT_TYPE } from '@/lib/workspace-files/page-compile' import { renderSimPageDocumentWithAssets } from '@/lib/workspace-files/page-document.server' -import { verifyFileAccess } from '@/app/api/files/authorization' +import { type KnowledgeFileAccess, verifyFileAccess } from '@/app/api/files/authorization' import type { UserFile } from '@/executor/types' const logger = createLogger('FileUtilsServer') @@ -229,6 +229,13 @@ export interface DownloadFileFromUrlOptions { * be treated as implicitly trusted. */ userId?: string + /** + * How a knowledge-base file identifies its reader. Omitted, the read is + * authorized as the workspace, which is what a caller-supplied URL gets. A + * background job processing a connector-owned row passes the system scope, + * because that row is hidden until the sync materializes who may read it. + */ + knowledgeAccess?: KnowledgeFileAccess } /** @@ -248,7 +255,13 @@ export async function downloadFileFromUrl( fileUrl: string, options: DownloadFileFromUrlOptions = {} ): Promise { - const { timeoutMs = getMaxExecutionTimeout(), maxBytes, signal, userId } = options + const { + timeoutMs = getMaxExecutionTimeout(), + maxBytes, + signal, + userId, + knowledgeAccess, + } = options signal?.throwIfAborted() @@ -266,7 +279,9 @@ export async function downloadFileFromUrl( const context = inferContextFromKey(key) - const hasAccess = await verifyFileAccess(key, userId, undefined, context, false) + const hasAccess = await verifyFileAccess(key, userId, undefined, context, false, { + knowledgeAccess, + }) if (!hasAccess) { logger.warn('Internal file download denied: access check failed', { key, context, userId }) throw new Error('Access denied: file not found or insufficient permissions') diff --git a/apps/sim/lib/uploads/utils/user-file-base64.server.test.ts b/apps/sim/lib/uploads/utils/user-file-base64.server.test.ts index df053676271..c9fe94f0725 100644 --- a/apps/sim/lib/uploads/utils/user-file-base64.server.test.ts +++ b/apps/sim/lib/uploads/utils/user-file-base64.server.test.ts @@ -9,26 +9,32 @@ import { } from '@/lib/uploads/utils/user-file-base64.server' import type { UserFile } from '@/executor/types' -const { mockDownloadFile, mockDownloadServableFileFromStorage, mockRedis, mockVerifyFileAccess } = - vi.hoisted(() => { - const mockRedis = { - get: vi.fn(), - set: vi.fn(), - hget: vi.fn(), - hset: vi.fn(), - hgetall: vi.fn(), - expire: vi.fn(), - scan: vi.fn(), - del: vi.fn(), - eval: vi.fn(), - } - return { - mockDownloadFile: vi.fn(), - mockDownloadServableFileFromStorage: vi.fn(), - mockRedis, - mockVerifyFileAccess: vi.fn(), - } - }) +const { + mockDownloadFile, + mockDownloadServableFileFromStorage, + mockRedis, + mockVerifyFileAccess, + mockResolveKnowledgeAccessScope, +} = vi.hoisted(() => { + const mockRedis = { + get: vi.fn(), + set: vi.fn(), + hget: vi.fn(), + hset: vi.fn(), + hgetall: vi.fn(), + expire: vi.fn(), + scan: vi.fn(), + del: vi.fn(), + eval: vi.fn(), + } + return { + mockDownloadFile: vi.fn(), + mockDownloadServableFileFromStorage: vi.fn(), + mockRedis, + mockVerifyFileAccess: vi.fn(), + mockResolveKnowledgeAccessScope: vi.fn(), + } +}) const mockGetRedisClient = redisConfigMockFns.mockGetRedisClient @@ -53,6 +59,10 @@ vi.mock('@/app/api/files/authorization', () => ({ verifyFileAccess: mockVerifyFileAccess, })) +vi.mock('@/lib/knowledge/access/scope', () => ({ + resolveKnowledgeAccessScope: mockResolveKnowledgeAccessScope, +})) + describe('hydrateUserFilesWithBase64', () => { beforeEach(() => { vi.clearAllMocks() @@ -240,6 +250,46 @@ describe('hydrateUserFilesWithBase64', () => { expect(hydrated.file).not.toHaveProperty('base64') }) + it('reads a knowledge-base file as the principal behind the run', async () => { + mockDownloadFile.mockResolvedValueOnce(Buffer.from('hello', 'utf8')) + const principal = { kind: 'session' as const, userId: 'user-1', sessionId: 'session-1' } + const scope = { kind: 'user' as const, tokens: ['user:user-1'] } + mockResolveKnowledgeAccessScope.mockResolvedValue(scope) + const file: UserFile = { + id: 'file-1', + name: 'shared.txt', + key: 'kb/workspace/shared.txt', + url: '/api/files/serve/kb/workspace/shared.txt?context=knowledge-base', + size: 5, + type: 'text/plain', + context: 'knowledge-base', + } + + const hydrated = await hydrateUserFilesWithBase64( + { file }, + { + workspaceId: 'workspace', + workflowId: 'workflow', + userId: 'user-1', + principal, + maxBytes: 10, + } + ) + + expect(hydrated.file.base64).toBe(Buffer.from('hello').toString('base64')) + expect(mockResolveKnowledgeAccessScope).toHaveBeenCalledWith(principal, { + workspaceId: 'workspace', + }) + expect(mockVerifyFileAccess).toHaveBeenCalledWith( + file.key, + 'user-1', + undefined, + 'knowledge-base', + false, + { knowledgeAccess: scope } + ) + }) + it('hydrates prior-execution files when workflow-scoped reads are enabled', async () => { mockDownloadFile.mockResolvedValueOnce(Buffer.from('hello', 'utf8')) const file: UserFile = { diff --git a/apps/sim/lib/uploads/utils/user-file-base64.server.ts b/apps/sim/lib/uploads/utils/user-file-base64.server.ts index cf2103bbb0e..87921a31836 100644 --- a/apps/sim/lib/uploads/utils/user-file-base64.server.ts +++ b/apps/sim/lib/uploads/utils/user-file-base64.server.ts @@ -1,3 +1,4 @@ +import type { Principal } from '@sim/auth/principal' import type { Logger } from '@sim/logger' import { createLogger } from '@sim/logger' import { isPlainRecord } from '@sim/utils/object' @@ -163,6 +164,13 @@ export interface Base64HydrationOptions { fileKeys?: string[] allowLargeValueWorkflowScope?: boolean userId?: string + /** + * The principal behind the run. A knowledge-base file is read as them, so a + * document shared with only this person still hydrates; `userId` alone may + * be the workflow owner standing in for an actorless run and must not widen + * what the run can read. + */ + principal?: Principal logger?: Logger maxBytes?: number allowUnknownSize?: boolean @@ -454,6 +462,7 @@ async function resolveBase64( fileKeys: options.fileKeys, allowLargeValueWorkflowScope: options.allowLargeValueWorkflowScope, userId: options.userId, + principal: options.principal, encoding: 'base64', maxBytes, }) @@ -487,6 +496,7 @@ async function hydrateUserFile( fileKeys: options.fileKeys, allowLargeValueWorkflowScope: options.allowLargeValueWorkflowScope, userId: options.userId, + principal: options.principal, logger, }) } catch (error) { diff --git a/apps/sim/lib/workflows/executor/execute-service.ts b/apps/sim/lib/workflows/executor/execute-service.ts index 3ebb700717f..7edae7f8b2c 100644 --- a/apps/sim/lib/workflows/executor/execute-service.ts +++ b/apps/sim/lib/workflows/executor/execute-service.ts @@ -523,6 +523,7 @@ export async function executeWorkflowService( workspaceId, workflowId, userId: actorUserId, + principal, allowLargeValueWorkflowScope: false, requestSignal: abortSignal, requestHeaders: headers, @@ -695,6 +696,7 @@ export async function executeWorkflowService( fileKeys: result.metadata?.fileKeys ?? [], allowLargeValueWorkflowScope: false, userId: actorUserId, + principal, maxBytes: base64MaxBytes, preserveLargeValueMetadata: true, })) as NormalizedBlockOutput) diff --git a/apps/sim/lib/workflows/streaming/streaming-principal.test.ts b/apps/sim/lib/workflows/streaming/streaming-principal.test.ts new file mode 100644 index 00000000000..b74432043c7 --- /dev/null +++ b/apps/sim/lib/workflows/streaming/streaming-principal.test.ts @@ -0,0 +1,82 @@ +/** + * @vitest-environment node + */ +import type { WorkflowExecutionPrincipal } from '@sim/auth/principal' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { createStreamingResponse } from '@/lib/workflows/streaming/streaming' + +const { mockNavigatePathAsync } = vi.hoisted(() => ({ + mockNavigatePathAsync: vi.fn(), +})) + +vi.mock('@/executor/variables/resolvers/reference-async.server', () => ({ + navigatePathAsync: mockNavigatePathAsync, +})) + +const principal = { + kind: 'session', + userId: 'user-1', +} as unknown as WorkflowExecutionPrincipal + +async function drain(stream: ReadableStream): Promise { + const reader = stream.getReader() + try { + while (!(await reader.read()).done) { + /* consume */ + } + } finally { + reader.releaseLock() + } +} + +describe('selected output principal', () => { + beforeEach(() => { + mockNavigatePathAsync.mockReset() + mockNavigatePathAsync.mockImplementation(async (value: unknown, path: string[]) => + path.reduce( + (current, part) => (current as Record | undefined)?.[part], + value + ) + ) + }) + + /** + * A block that completes without the selected path streams nothing, so the + * output is materialized from the final result instead. Both reads must run + * as the principal behind the run, or a member-only knowledge-base file in + * the output is read as nobody on the final-frame path. + */ + it('reads a selected output as the executing principal on the chunk and final paths', async () => { + const stream = await createStreamingResponse({ + requestId: 'request-1', + principal, + streamConfig: { selectedOutputs: ['agent_content'] }, + executeFn: async ({ onBlockComplete }) => { + await onBlockComplete('agent', {}) + const output = { content: 'Done' } + return { + success: true, + output, + logs: [ + { + blockId: 'agent', + output, + startedAt: new Date().toISOString(), + endedAt: new Date().toISOString(), + durationMs: 1, + success: true, + }, + ], + } as never + }, + }) + + await drain(stream) + + expect(mockNavigatePathAsync).toHaveBeenCalledTimes(2) + for (const [, path, context] of mockNavigatePathAsync.mock.calls) { + expect(path).toEqual(['content']) + expect(context.executionContext.principal).toBe(principal) + } + }) +}) diff --git a/apps/sim/lib/workflows/streaming/streaming.ts b/apps/sim/lib/workflows/streaming/streaming.ts index dda38e49dab..7c57eb35586 100644 --- a/apps/sim/lib/workflows/streaming/streaming.ts +++ b/apps/sim/lib/workflows/streaming/streaming.ts @@ -1,3 +1,4 @@ +import type { WorkflowExecutionPrincipal } from '@sim/auth/principal' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { isRecordLike, omit } from '@sim/utils/object' @@ -95,6 +96,8 @@ export interface StreamingResponseOptions { workspaceId?: string workflowId?: string userId?: string + /** The principal behind the run; knowledge-base files in the output are read as them. */ + principal?: WorkflowExecutionPrincipal /** Incoming fetch/request abort — combined with the stream timeout. */ requestSignal?: AbortSignal /** Used with the independent event policies to negotiate agent-events SSE. */ @@ -155,6 +158,7 @@ type OutputExtractionContext = Pick< | 'fileKeys' | 'allowLargeValueWorkflowScope' | 'userId' + | 'principal' > & { base64MaxBytes?: number } async function extractOutputValue( @@ -174,6 +178,7 @@ async function extractOutputValue( fileKeys: context.fileKeys, allowLargeValueWorkflowScope: context.allowLargeValueWorkflowScope, userId: context.userId, + principal: context.principal, metadata: { requestId: context.requestId }, base64MaxBytes: context.base64MaxBytes, }, @@ -225,6 +230,7 @@ function buildMaterializationContext( fileKeys: context.fileKeys, allowLargeValueWorkflowScope: context.allowLargeValueWorkflowScope, userId: context.userId, + principal: context.principal, } } @@ -748,6 +754,7 @@ export async function createStreamingResponse( fileKeys: options.fileKeys, allowLargeValueWorkflowScope: options.allowLargeValueWorkflowScope, userId: options.userId, + principal: options.principal, base64MaxBytes: Math.min( base64MaxBytes ?? MAX_INLINE_MATERIALIZATION_BYTES, getBase64DecodedByteBudget(remainingBytes) @@ -879,6 +886,7 @@ export async function createStreamingResponse( fileKeys: result.metadata?.fileKeys ?? options.fileKeys, allowLargeValueWorkflowScope: options.allowLargeValueWorkflowScope, userId: options.userId, + principal: options.principal, redactToolPayloads: streamConfig.isSecureMode === true, } ) diff --git a/apps/sim/lib/workspaces/host-context.ts b/apps/sim/lib/workspaces/host-context.ts index 6fdd788d90a..78cd140507f 100644 --- a/apps/sim/lib/workspaces/host-context.ts +++ b/apps/sim/lib/workspaces/host-context.ts @@ -2,6 +2,7 @@ import { cache } from 'react' import type { WorkspaceHostContext } from '@/lib/api/contracts/workspaces' import { getWorkspaceOwnerSubscriptionAccess } from '@/lib/billing/core/workspace-access' import { isCredentialGroupsAvailable } from '@/lib/credential-groups/availability' +import { isKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' import { getOrganizationSettingsAccess } from '@/lib/organizations/settings-access' import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils' @@ -28,7 +29,10 @@ async function resolveWorkspaceHostContextForViewer( ? getOrganizationSettingsAccess(hostOrganizationId, userId) : Promise.resolve({ role: null, isMember: false, isAdmin: false }), ]) - const credentialGroupsAvailable = await isCredentialGroupsAvailable({ workspaceId, ownerBilling }) + const [credentialGroupsAvailable, knowledgeMemberAccessAvailable] = await Promise.all([ + isCredentialGroupsAvailable({ workspaceId, ownerBilling }), + isKnowledgeMemberAccessAvailable({ workspaceId, ownerBilling }), + ]) return { workspace: { @@ -48,6 +52,7 @@ async function resolveWorkspaceHostContextForViewer( }, features: { credentialGroups: credentialGroupsAvailable, + knowledgeMemberAccess: knowledgeMemberAccessAvailable, }, } } diff --git a/apps/sim/stores/mothership-queue/store.ts b/apps/sim/stores/mothership-queue/store.ts index 4ecdbb89c61..2bd38b4425a 100644 --- a/apps/sim/stores/mothership-queue/store.ts +++ b/apps/sim/stores/mothership-queue/store.ts @@ -112,6 +112,7 @@ export const useMothershipQueueStore = create()( content: patch.content, fileAttachments: patch.fileAttachments, contexts: patch.contexts, + requestMode: patch.requestMode, } return { queues: setQueueForChat(state.queues, chatKey, next) } }), diff --git a/apps/sim/stores/mothership-queue/types.ts b/apps/sim/stores/mothership-queue/types.ts index b7beb4ede09..9d2267b9ac1 100644 --- a/apps/sim/stores/mothership-queue/types.ts +++ b/apps/sim/stores/mothership-queue/types.ts @@ -21,7 +21,10 @@ export type QueuedMothershipMessage = QueuedMessage & { } // Mutable fields an in-place edit overwrites; id and index are preserved by `replaceAt`. -export type QueuedMessageEditPatch = Pick +export type QueuedMessageEditPatch = Pick< + QueuedMessage, + 'content' | 'fileAttachments' | 'contexts' | 'requestMode' +> export interface MothershipQueueState { queues: Record diff --git a/apps/sim/tools/generated/tool-metadata.ts b/apps/sim/tools/generated/tool-metadata.ts index d699e4f8330..7cfd07be9c9 100644 --- a/apps/sim/tools/generated/tool-metadata.ts +++ b/apps/sim/tools/generated/tool-metadata.ts @@ -3,7 +3,7 @@ /** Serializable metadata for every built-in tool, keyed by tool id. */ const toolMetadata: Record = JSON.parse( - '{"a2a_cancel_task":{"id":"a2a_cancel_task","name":"A2A Cancel Task","description":"Request cancellation of an in-progress A2A task.","version":"1.0.0","params":{"agentUrl":{"type":"string","required":true,"visibility":"user-only","description":"The A2A agent endpoint URL"},"taskId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The task ID to cancel"},"apiKey":{"type":"string","required":false,"visibility":"user-only","description":"API key for authentication (if required)"}},"hostedApiKey":"none"},"a2a_get_agent_card":{"id":"a2a_get_agent_card","name":"A2A Get Agent Card","description":"Fetch the Agent Card (discovery document) for an external A2A agent.","version":"1.0.0","params":{"agentUrl":{"type":"string","required":true,"visibility":"user-only","description":"The A2A agent endpoint URL"},"apiKey":{"type":"string","required":false,"visibility":"user-only","description":"API key for authentication (if required)"}},"hostedApiKey":"none"},"a2a_get_task":{"id":"a2a_get_task","name":"A2A Get Task","description":"Retrieve the current state and result of an A2A task.","version":"1.0.0","params":{"agentUrl":{"type":"string","required":true,"visibility":"user-only","description":"The A2A agent endpoint URL"},"taskId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The task ID to retrieve"},"historyLength":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of history messages to include"},"apiKey":{"type":"string","required":false,"visibility":"user-only","description":"API key for authentication (if required)"}},"hostedApiKey":"none"},"a2a_send_message":{"id":"a2a_send_message","name":"A2A Send Message","description":"Send a message to an external A2A agent and return its response.","version":"1.0.0","params":{"agentUrl":{"type":"string","required":true,"visibility":"user-only","description":"The A2A agent endpoint URL"},"message":{"type":"string","required":true,"visibility":"user-or-llm","description":"The message text to send"},"data":{"type":"json","required":false,"visibility":"user-or-llm","description":"Optional structured JSON data to attach"},"files":{"type":"json","required":false,"visibility":"user-or-llm","description":"Optional files to attach"},"taskId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Existing task ID to continue"},"contextId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Conversation context ID to continue"},"apiKey":{"type":"string","required":false,"visibility":"user-only","description":"API key for authentication (if required)"}},"hostedApiKey":"none"},"affinity_batch_update_entity_fields":{"id":"affinity_batch_update_entity_fields","name":"Affinity Batch Update Entity Fields","description":"Write up to 100 non-list field values on one company or person in a single request.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which entity to write the fields on: companies or persons"},"entityId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of that company or person"},"updates":{"type":"json","required":true,"visibility":"user-or-llm","description":"Up to 100 field updates as [{\\"id\\":\\"\\",\\"value\\":{\\"type\\":\\"…\\",\\"data\\":…}}], using the same value shapes as a single field update"}},"hostedApiKey":"none"},"affinity_batch_update_list_entry_fields":{"id":"affinity_batch_update_list_entry_fields","name":"Affinity Batch Update List Entry Fields","description":"Write up to 100 field values on one list row in a single request. Requires the \\"Export data from Lists\\" permission.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"listEntryId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list entry ID"},"updates":{"type":"json","required":true,"visibility":"user-or-llm","description":"Up to 100 field updates as [{\\"id\\":\\"\\",\\"value\\":{\\"type\\":\\"…\\",\\"data\\":…}}], using the same value shapes as a single field update"}},"hostedApiKey":"none"},"affinity_create_list":{"id":"affinity_create_list","name":"Affinity Create List","description":"Create a list. Its type fixes which entities it can hold, and the API key holder becomes its creator and owner.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the new list"},"type":{"type":"string","required":true,"visibility":"user-or-llm","description":"Entity kind the list holds: company, opportunity, or person"},"isPublic":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Whether everyone in the organization can see the list"}},"hostedApiKey":"none"},"affinity_create_list_field_dropdown_option":{"id":"affinity_create_list_field_dropdown_option","name":"Affinity Create List Field Dropdown Option","description":"Add a selectable option to a dropdown field on a list. A ranked or status option also needs a rank and a color.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"fieldId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The dropdown field ID on that list"},"type":{"type":"string","required":true,"visibility":"user-or-llm","description":"Kind of option to create, matching the field. dropdown takes only a label; ranked-dropdown also requires rank and color; status-dropdown additionally requires a status category. Sending a field the kind does not accept is rejected"},"text":{"type":"string","required":true,"visibility":"user-or-llm","description":"The option label"},"rank":{"type":"number","required":false,"visibility":"user-or-llm","description":"Sort order. Required on a ranked-dropdown or status-dropdown option"},"color":{"type":"string","required":false,"visibility":"user-or-llm","description":"Option color: white, gray, blue, green, purple, orange, or red. Required on a ranked-dropdown or status-dropdown option"},"statusCategory":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pipeline meaning of the option: open, won, lost, or on-hold. Status-dropdown options only"},"winRate":{"type":"number","required":false,"visibility":"user-or-llm","description":"Expected win rate of the status. Status-dropdown options only"}},"hostedApiKey":"none"},"affinity_create_merge":{"id":"affinity_create_merge","name":"Affinity Create Merge","description":"Fold a duplicate company or person into the record you are keeping. The merge runs asynchronously — poll the returned task to see it finish. Requires the \\"Manage duplicates\\" permission and an admin role.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"What to merge: companies or persons"},"primaryId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record to keep"},"duplicateId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the duplicate record to fold in"}},"hostedApiKey":"none"},"affinity_create_note":{"id":"affinity_create_note","name":"Affinity Create Note","description":"Write a note — attached to companies, persons, and opportunities, anchored to a meeting, call, or chat message, or posted as a reply to an existing note.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"type":{"type":"string","required":true,"visibility":"user-or-llm","description":"Note shape: entities to attach it to records, interaction to anchor it to a meeting, call, or chat message, or user-reply to reply to a note"},"html":{"type":"string","required":true,"visibility":"user-or-llm","description":"The note body as HTML"},"companyIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Companies to attach the note to, e.g. [1, 2]. Not used on a reply"},"personIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Persons to attach the note to, e.g. [1, 2]. Not used on a reply"},"opportunityIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Opportunities to attach the note to, e.g. [1, 2]. Not used on a reply"},"interactionId":{"type":"string","required":false,"visibility":"user-or-llm","description":"The interaction to anchor the note to. Required for an interaction note"},"interactionType":{"type":"string","required":false,"visibility":"user-or-llm","description":"Kind of the anchoring interaction: meeting, call, or chat-message. Required for an interaction note"},"parentId":{"type":"string","required":false,"visibility":"user-or-llm","description":"The note being replied to. Required for a user-reply note"},"creatorId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Attribute the note to another internal person. Defaults to the API key holder"},"createdAt":{"type":"string","required":false,"visibility":"user-or-llm","description":"Backdate the note to this ISO 8601 timestamp"}},"hostedApiKey":"none"},"affinity_create_reminder":{"id":"affinity_create_reminder","name":"Affinity Create Reminder","description":"Create a reminder on one company, person, or opportunity. A recurring reminder resets whenever the chosen signal happens instead of firing once.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"type":{"type":"string","required":true,"visibility":"user-or-llm","description":"one-time to fire once, or recurring to reset on a signal"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"What the reminder is about: company, person, or opportunity"},"entityId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of that company, person, or opportunity"},"dueDate":{"type":"string","required":false,"visibility":"user-or-llm","description":"When the reminder is due, as an ISO 8601 timestamp. Required for a one-time reminder; on a recurring one Affinity computes it from the period when omitted"},"content":{"type":"string","required":false,"visibility":"user-or-llm","description":"What the reminder says"},"ownerId":{"type":"string","required":true,"visibility":"user-or-llm","description":"User the reminder is assigned to. Must be an internal user. The API key holder is recorded as the creator, which is a separate field"},"resetTrigger":{"type":"string","required":false,"visibility":"user-or-llm","description":"What restarts a recurring reminder: interaction, email, or event. Required when the type is recurring"},"periodDays":{"type":"number","required":false,"visibility":"user-or-llm","description":"Days between firings of a recurring reminder. Required when the type is recurring"}},"hostedApiKey":"none"},"affinity_delete_list_field_dropdown_option":{"id":"affinity_delete_list_field_dropdown_option","name":"Affinity Delete List Field Dropdown Option","description":"Permanently delete a dropdown option on a list field. Every list entry currently set to it is cleared, and those values cannot be recovered.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"fieldId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The dropdown field ID on that list"},"dropdownOptionId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The dropdown option ID to delete"}},"hostedApiKey":"none"},"affinity_delete_note":{"id":"affinity_delete_note","name":"Affinity Delete Note","description":"Delete a note you created. Deleting a root note also deletes its replies; deleting a reply removes only that reply.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"noteId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The note ID to delete"}},"hostedApiKey":"none"},"affinity_get_company":{"id":"affinity_get_company","name":"Affinity Get Company","description":"Look up one company by ID. Field data is returned only for the Field IDs or Field Types asked for.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"companyId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The company ID"},"fieldIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field IDs to return values for, e.g. [\\"affinity-data-location\\"]. Mutually exclusive with Field Types"},"fieldTypes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field categories to return values for: enriched, global, or relationship-intelligence. Mutually exclusive with Field IDs"}},"hostedApiKey":"none"},"affinity_get_current_user":{"id":"affinity_get_current_user","name":"Affinity Get Current User","description":"Verify an Affinity API key and return the tenant, the user behind the key, and the scopes the grant carries.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"}},"hostedApiKey":"none"},"affinity_get_entity_field_value":{"id":"affinity_get_entity_field_value","name":"Affinity Get Entity Field Value","description":"Read one non-list field value from a company or person.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which entity to read the field from: companies or persons"},"entityId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of that company or person"},"fieldId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The field ID to read"}},"hostedApiKey":"none"},"affinity_get_list":{"id":"affinity_get_list","name":"Affinity Get List","description":"Read one list — its name, type, owner, and privacy setting.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"}},"hostedApiKey":"none"},"affinity_get_list_entry":{"id":"affinity_get_list_entry","name":"Affinity Get List Entry","description":"Read one row of a list with its entity. Field data is returned only for the Field IDs or Field Types asked for.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"listEntryId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list entry ID"},"fieldIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field IDs to return values for, e.g. [\\"affinity-data-location\\"]. Mutually exclusive with Field Types"},"fieldTypes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field categories to return values for: enriched, global, list, or relationship-intelligence. Mutually exclusive with Field IDs"}},"hostedApiKey":"none"},"affinity_get_list_entry_field":{"id":"affinity_get_list_entry_field","name":"Affinity Get List Entry Field","description":"Read one field value on a list row.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"listEntryId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list entry ID"},"fieldId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The field ID to read"}},"hostedApiKey":"none"},"affinity_get_list_field_dropdown_option":{"id":"affinity_get_list_field_dropdown_option","name":"Affinity Get List Field Dropdown Option","description":"Read one dropdown option on a list field.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"fieldId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The dropdown field ID on that list"},"dropdownOptionId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The dropdown option ID"}},"hostedApiKey":"none"},"affinity_get_merge":{"id":"affinity_get_merge","name":"Affinity Get Merge","description":"Read the status of one company or person merge, including why it failed if it did.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which merge to read: companies or persons"},"mergeId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The merge ID"}},"hostedApiKey":"none"},"affinity_get_merge_task":{"id":"affinity_get_merge_task","name":"Affinity Get Merge Task","description":"Read one merge task and how its merges are progressing. Poll this after starting a merge.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which merge task to read: companies or persons"},"taskId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The merge task ID"}},"hostedApiKey":"none"},"affinity_get_note":{"id":"affinity_get_note","name":"Affinity Get Note","description":"Read one note with its body, author, mentions, and attached records.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"noteId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The note ID"},"includes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Extra properties to return, e.g. [\\"repliesCount\\",\\"personsPreview\\",\\"companiesPreview\\",\\"opportunitiesPreview\\"]. Those four fields are omitted unless requested here"}},"hostedApiKey":"none"},"affinity_get_opportunity":{"id":"affinity_get_opportunity","name":"Affinity Get Opportunity","description":"Read one opportunity and the list it belongs to. Its field data lives on the list entry.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"opportunityId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The opportunity ID"}},"hostedApiKey":"none"},"affinity_get_person":{"id":"affinity_get_person","name":"Affinity Get Person","description":"Look up one person by ID. Field data is returned only for the Field IDs or Field Types asked for.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"personId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The person ID"},"fieldIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field IDs to return values for, e.g. [\\"affinity-data-location\\"]. Mutually exclusive with Field Types"},"fieldTypes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field categories to return values for: enriched, global, or relationship-intelligence. Mutually exclusive with Field IDs"}},"hostedApiKey":"none"},"affinity_get_saved_view":{"id":"affinity_get_saved_view","name":"Affinity Get Saved View","description":"Read one saved view — its name, kind, and creation date.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"viewId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The saved view ID"}},"hostedApiKey":"none"},"affinity_get_transcript":{"id":"affinity_get_transcript","name":"Affinity Get Transcript","description":"Read one transcript with its first 100 fragments. Page the fragments endpoint for a longer meeting.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"transcriptId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The transcript ID"}},"hostedApiKey":"none"},"affinity_get_user":{"id":"affinity_get_user","name":"Affinity Get User","description":"Read one internal user. A user and their person record share the same numeric ID, so a person ID works here.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"userId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The user ID, which is also their person ID"}},"hostedApiKey":"none"},"affinity_list_calls":{"id":"affinity_list_calls","name":"Affinity List Calls","description":"Page through logged calls and their participants. Only calls the API key holder can see are returned.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression, e.g. \\"createdAt>=2026-01-01\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_chat_messages":{"id":"affinity_list_chat_messages","name":"Affinity List Chat Messages","description":"Page through logged chat messages and their participants. Only messages the API key holder can see are returned.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression, e.g. \\"createdAt>=2026-01-01\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_companies":{"id":"affinity_list_companies","name":"Affinity List Companies","description":"Page through companies. Companies come back without field data unless Field IDs or Field Types asks for it.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"ids":{"type":"json","required":false,"visibility":"user-or-llm","description":"Restrict the page to these company IDs, e.g. [1, 2, 3]"},"fieldIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field IDs to return values for, e.g. [\\"affinity-data-location\\"]. Mutually exclusive with Field Types"},"fieldTypes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field categories to return values for: enriched, global, or relationship-intelligence. Mutually exclusive with Field IDs"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_coworker_connections":{"id":"affinity_list_coworker_connections","name":"Affinity List Coworker Connections","description":"Find warm paths into a company through shared work history: who in your Affinity data once worked alongside the people you want to reach. Grouped by target, strongest first.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"filter":{"type":"string","required":true,"visibility":"user-or-llm","description":"Required scope. The only supported filter is target.currentCompany.id, e.g. \\"target.currentCompany.id=123\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of targets to return per page, 1-50. Defaults to 20"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_list_emails":{"id":"affinity_list_emails","name":"Affinity List Emails","description":"Page through email metadata — subject, participants, and timestamps. Affinity never exposes email bodies through the API.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression, e.g. \\"createdAt>=2026-01-01\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_entity_field_values":{"id":"affinity_list_entity_field_values","name":"Affinity List Entity Field Values","description":"Page through a company\'s or person\'s non-list field values. List fields are not returned here — read those through the list entry.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which entity to read field values from: companies or persons"},"entityId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of that company or person"},"ids":{"type":"json","required":false,"visibility":"user-or-llm","description":"Restrict to these field IDs. Mutually exclusive with Field Types"},"types":{"type":"json","required":false,"visibility":"user-or-llm","description":"Restrict to these field categories: enriched, global, relationship-intelligence. Mutually exclusive with Field IDs"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 20"}},"hostedApiKey":"none"},"affinity_list_entity_list_entries":{"id":"affinity_list_entity_list_entries","name":"Affinity List Entity List Entries","description":"Page through a company\'s or person\'s rows across every list, each carrying that list\'s field values and when the entity was added.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which entity to look up the rows of: companies or persons"},"entityId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of that company or person"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_entity_lists":{"id":"affinity_list_entity_lists","name":"Affinity List Entity Lists","description":"List every list a company or person appears on that the caller can view.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which entity to look up the lists of: companies or persons"},"entityId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of that company or person"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_entity_notes":{"id":"affinity_list_entity_notes","name":"Affinity List Entity Notes","description":"List the notes relevant to one company, person, or opportunity — directly attached notes plus notes reaching it through its people and meetings.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which entity the notes hang off: companies, persons, or opportunities"},"entityId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of that company, person, or opportunity"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression, e.g. \\"createdAt>=2026-01-01\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_list_entity_relationships":{"id":"affinity_list_entity_relationships","name":"Affinity List Entity Relationships","description":"List who knows a company or person, scored 0.0 to 1.0 by how much the two actually interact. Strongest first by default.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which entity to look up relationships for: companies or persons"},"entityId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of that company or person"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression. This endpoint filters on interactionScore only, e.g. \\"interactionScore>=0.5\\""},"orderBy":{"type":"json","required":false,"visibility":"user-or-llm","description":"Sort order: [\\"interactionScore\\"] for weakest first, [\\"-interactionScore\\"] for strongest first (the default)"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_list_field_dropdown_options":{"id":"affinity_list_field_dropdown_options","name":"Affinity List Field Dropdown Options","description":"List the selectable options on a dropdown or ranked-dropdown company or person field. Writing such a field needs the option ID, not its text.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which field family the field belongs to: companies or persons"},"fieldId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The dropdown or ranked-dropdown field ID"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_field_metadata":{"id":"affinity_list_field_metadata","name":"Affinity List Field Metadata","description":"List the non-list company or person fields, with the value type, filter operators, and sort support of each. Start here to find the Field IDs the read and write tools take.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which fields to describe: companies or persons"},"includes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Extra properties to return: [\\"filterability\\",\\"sortability\\"]. Both are omitted unless requested here"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression. This endpoint filters on name only, e.g. \\"name=~Status\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_field_value_changes":{"id":"affinity_list_field_value_changes","name":"Affinity List Field Value Changes","description":"Page through field value changes across the whole workspace. Built for delta sync: follow nextCursor to the end of a run, then resume from the last cursor next time.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression over field.id, listEntry.id, changer.id, changedAt, or actionType. Resume a sync with e.g. \\"changedAt>2026-06-01T12:00:00Z\\""},"orderBy":{"type":"json","required":false,"visibility":"user-or-llm","description":"Sort order: [\\"changedAt\\"] for oldest first (the default), [\\"-changedAt\\"] for newest first"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_investor_executive_connections":{"id":"affinity_list_investor_executive_connections","name":"Affinity List Investor Executive Connections","description":"Find warm paths into a company through investment history: which investors in your Affinity data backed a company the people you want to reach once led. Grouped by target, strongest first.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"filter":{"type":"string","required":true,"visibility":"user-or-llm","description":"Required scope. The only supported filter is target.currentCompany.id, e.g. \\"target.currentCompany.id=123\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of targets to return per page, 1-50. Defaults to 20"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_list_list_entries":{"id":"affinity_list_list_entries","name":"Affinity List List Entries","description":"Page through the rows of a list. Rows come back without field data unless Field IDs or Field Types asks for it.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"fieldIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field IDs to return values for, e.g. [\\"affinity-data-location\\"]. Mutually exclusive with Field Types"},"fieldTypes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field categories to return values for: enriched, global, list, or relationship-intelligence. Mutually exclusive with Field IDs"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_list_entry_field_value_changes":{"id":"affinity_list_list_entry_field_value_changes","name":"Affinity List List Entry Field Value Changes","description":"Page through the history of one list row — who changed which field, when, and to what.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"listEntryId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list entry ID"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression over field.id, changer.id, changedAt, or actionType, e.g. \\"field.id=field-1234\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_list_entry_fields":{"id":"affinity_list_list_entry_fields","name":"Affinity List List Entry Fields","description":"Page through every field value on one list row, including the list-specific columns. All fields are returned unless narrowed.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"listEntryId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list entry ID"},"ids":{"type":"json","required":false,"visibility":"user-or-llm","description":"Restrict to these field IDs. Mutually exclusive with Field Types"},"types":{"type":"json","required":false,"visibility":"user-or-llm","description":"Restrict to these field categories: enriched, global, list, relationship-intelligence. Mutually exclusive with Field IDs"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 20"}},"hostedApiKey":"none"},"affinity_list_list_field_dropdown_options":{"id":"affinity_list_list_field_dropdown_options","name":"Affinity List List Field Dropdown Options","description":"List the selectable options on a dropdown, ranked-dropdown, or status-dropdown field of a list.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"fieldId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The dropdown field ID on that list"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_list_fields":{"id":"affinity_list_list_fields","name":"Affinity List List Fields","description":"List the fields available on one list, including its list-specific columns. Use these Field IDs when reading or writing list entries.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"includes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Extra properties to return: [\\"filterability\\",\\"sortability\\"]. Both are omitted unless requested here"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression. This endpoint filters on name only, e.g. \\"name=~Stage\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_lists":{"id":"affinity_list_lists","name":"Affinity List Lists","description":"Page through the lists in the organization that the caller can view.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"term":{"type":"string","required":false,"visibility":"user-or-llm","description":"Case-insensitive substring match on the list name"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_meetings":{"id":"affinity_list_meetings","name":"Affinity List Meetings","description":"Page through past and upcoming meetings with their organizer and attendees.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression, e.g. \\"createdAt>=2026-01-01\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_merge_tasks":{"id":"affinity_list_merge_tasks","name":"Affinity List Merge Tasks","description":"Page through merge tasks, each summarizing how many of its merges are in progress, succeeded, or failed.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which merge tasks to list: companies or persons"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression. This endpoint filters on status only, e.g. \\"status=in-progress\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_merges":{"id":"affinity_list_merges","name":"Affinity List Merges","description":"Page through the company or person merges the organization has run, with the status and the records involved in each.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which merges to list: companies or persons"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression over status or taskId, e.g. \\"status=failed\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_note_attached_companies":{"id":"affinity_list_note_attached_companies","name":"Affinity List Note Attached Companies","description":"List the companies directly attached to one note.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"noteId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The note ID"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_list_note_attached_opportunities":{"id":"affinity_list_note_attached_opportunities","name":"Affinity List Note Attached Opportunities","description":"List the opportunities directly attached to one note.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"noteId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The note ID"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_list_note_attached_persons":{"id":"affinity_list_note_attached_persons","name":"Affinity List Note Attached Persons","description":"List the persons directly attached to one note.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"noteId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The note ID"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_list_note_replies":{"id":"affinity_list_note_replies","name":"Affinity List Note Replies","description":"Page through the replies on one note, including AI Notetaker replies.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"noteId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The note ID whose replies to read"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression, e.g. \\"createdAt>=2026-01-01\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_list_notes":{"id":"affinity_list_notes","name":"Affinity List Notes","description":"Page through every note the caller can see. Replies are excluded.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"includes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Extra properties to return, e.g. [\\"repliesCount\\",\\"personsPreview\\",\\"companiesPreview\\",\\"opportunitiesPreview\\"]. Those four fields are omitted unless requested here"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression, e.g. \\"createdAt>=2026-01-01\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_list_opportunities":{"id":"affinity_list_opportunities","name":"Affinity List Opportunities","description":"Page through opportunities. Field data lives on the list entry, not here — read it through the list or saved view the opportunity belongs to.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"ids":{"type":"json","required":false,"visibility":"user-or-llm","description":"Restrict the page to these opportunity IDs, e.g. [1, 2, 3]"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_persons":{"id":"affinity_list_persons","name":"Affinity List Persons","description":"Page through persons. Persons come back without field data unless Field IDs or Field Types asks for it.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"ids":{"type":"json","required":false,"visibility":"user-or-llm","description":"Restrict the page to these person IDs, e.g. [1, 2, 3]"},"fieldIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field IDs to return values for, e.g. [\\"affinity-data-location\\"]. Mutually exclusive with Field Types"},"fieldTypes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field categories to return values for: enriched, global, or relationship-intelligence. Mutually exclusive with Field IDs"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_reminders":{"id":"affinity_list_reminders","name":"Affinity List Reminders","description":"Page through the reminders the caller can see. Filter by status to surface what is overdue.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression, e.g. \\"createdAt>=2026-01-01\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_list_saved_view_entries":{"id":"affinity_list_saved_view_entries","name":"Affinity List Saved View Entries","description":"Page through the rows of a saved view. The view\'s own filters and columns decide which rows and which field data come back.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"viewId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The saved view ID"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_saved_views":{"id":"affinity_list_saved_views","name":"Affinity List Saved Views","description":"List the saved views on a list that the caller can view.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_list_transcript_fragments":{"id":"affinity_list_transcript_fragments","name":"Affinity List Transcript Fragments","description":"Page through everything said in a meeting, segment by segment with the speaker.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"transcriptId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The transcript ID"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_list_transcripts":{"id":"affinity_list_transcripts","name":"Affinity List Transcripts","description":"Page through meeting transcript metadata. Read one transcript to get what was actually said.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression, e.g. \\"createdAt>=2026-01-01\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_list_users":{"id":"affinity_list_users","name":"Affinity List Users","description":"Page through the internal users in the organization. Email addresses and roles are returned only to callers with the \\"Manage Users\\" permission.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"term":{"type":"string","required":false,"visibility":"user-or-llm","description":"Case-insensitive match across first name, last name, and primary email"},"filter":{"type":"string","required":false,"visibility":"user-or-llm","description":"Affinity Filtering Language expression over id or status, e.g. \\"status=active\\""},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_search_companies":{"id":"affinity_search_companies","name":"Affinity Search Companies","description":"Search companies by filters, sorts, and a free-text term. Requires the \\"Export All Organizations directory\\" permission.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"filters":{"type":"json","required":false,"visibility":"user-or-llm","description":"Filter group as {operator: \\"and\\"|\\"or\\", filters: [...]}, at most 50 leaves. Each leaf is {valueType, fieldId, operator, value}, and a leaf may itself be a nested group"},"searchTerm":{"type":"string","required":false,"visibility":"user-or-llm","description":"Free-text term matched against the searchable fields. At least 3 characters"},"searchFieldIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field IDs the search term is matched against. Defaults to the searchable fields"},"sorts":{"type":"json","required":false,"visibility":"user-or-llm","description":"Sort order as [{fieldId, direction: \\"asc\\"|\\"desc\\", attributeId?}], up to 5, applied in order"},"fieldIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field IDs to return values for, e.g. [\\"affinity-data-location\\"]. Mutually exclusive with Field Types"},"fieldTypes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field categories to return values for: enriched, global, or relationship-intelligence. Mutually exclusive with Field IDs"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_search_files":{"id":"affinity_search_files","name":"Affinity Search Files","description":"Search files by keyword, ordered by relevance. Narrow to specific files or to one company, or leave both unset to search the whole account.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"prompt":{"type":"string","required":true,"visibility":"user-or-llm","description":"What to search for. Between 3 and 500 characters"},"ids":{"type":"json","required":false,"visibility":"user-or-llm","description":"Restrict the search to these file IDs. Cannot be combined with Company ID"},"companyId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Restrict the search to one company\'s files. Cannot be combined with file IDs"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of files to return, 1-100. Defaults to 20"}},"hostedApiKey":"none"},"affinity_search_list_entries":{"id":"affinity_search_list_entries","name":"Affinity Search List Entries","description":"Search the rows of one list by filters, sorts, and a free-text term. Requires the \\"Export data from Lists\\" permission.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID to search"},"filters":{"type":"json","required":false,"visibility":"user-or-llm","description":"Filter group as {operator: \\"and\\"|\\"or\\", filters: [...]}, at most 50 leaves. Each leaf is {valueType, fieldId, operator, value}, and a leaf may itself be a nested group"},"searchTerm":{"type":"string","required":false,"visibility":"user-or-llm","description":"Free-text term matched against the searchable fields. At least 3 characters"},"searchFieldIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field IDs the search term is matched against. Defaults to the searchable fields"},"sorts":{"type":"json","required":false,"visibility":"user-or-llm","description":"Sort order as [{fieldId, direction: \\"asc\\"|\\"desc\\", attributeId?}], up to 5, applied in order"},"fieldIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field IDs to return values for, e.g. [\\"affinity-data-location\\"]. Mutually exclusive with Field Types"},"fieldTypes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field categories to return values for: enriched, global, list, or relationship-intelligence. Mutually exclusive with Field IDs"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_search_notes":{"id":"affinity_search_notes","name":"Affinity Search Notes","description":"Search notes by keyword, ordered by relevance. Narrow to specific notes or to one company, or leave both unset to search the whole account.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"prompt":{"type":"string","required":true,"visibility":"user-or-llm","description":"What to search for. Between 3 and 500 characters"},"ids":{"type":"json","required":false,"visibility":"user-or-llm","description":"Restrict the search to these note IDs. Cannot be combined with Company ID"},"companyId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Restrict the search to one company\'s notes. Cannot be combined with note IDs"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of notes to return, 1-100. Defaults to 20"}},"hostedApiKey":"none"},"affinity_search_persons":{"id":"affinity_search_persons","name":"Affinity Search Persons","description":"Search persons by filters, sorts, and a free-text term. Requires the \\"Export All People directory\\" permission.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"filters":{"type":"json","required":false,"visibility":"user-or-llm","description":"Filter group as {operator: \\"and\\"|\\"or\\", filters: [...]}, at most 50 leaves. Each leaf is {valueType, fieldId, operator, value}, and a leaf may itself be a nested group"},"searchTerm":{"type":"string","required":false,"visibility":"user-or-llm","description":"Free-text term matched against the searchable fields. At least 3 characters"},"searchFieldIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field IDs the search term is matched against. Defaults to the searchable fields"},"sorts":{"type":"json","required":false,"visibility":"user-or-llm","description":"Sort order as [{fieldId, direction: \\"asc\\"|\\"desc\\", attributeId?}], up to 5, applied in order"},"fieldIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field IDs to return values for, e.g. [\\"affinity-data-location\\"]. Mutually exclusive with Field Types"},"fieldTypes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Field categories to return values for: enriched, global, or relationship-intelligence. Mutually exclusive with Field IDs"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous page, returned as nextCursor or prevCursor"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to return per page, 1-100. Defaults to 100"},"totalCount":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the total size of the collection. Costs an extra query"}},"hostedApiKey":"none"},"affinity_semantic_search":{"id":"affinity_semantic_search","name":"Affinity Semantic Search","description":"Find companies from a description in plain language — industry, technology, stage, or business model. Currently searches companies only.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"prompt":{"type":"string","required":true,"visibility":"user-or-llm","description":"What to look for, in plain language, e.g. \\"climate tech companies in our pipeline\\". Up to 500 characters"},"listIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Restrict the search to companies on these lists, e.g. [1, 2]"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of companies to return, 1-100. Defaults to 100"}},"hostedApiKey":"none"},"affinity_update_entity_field_value":{"id":"affinity_update_entity_field_value","name":"Affinity Update Entity Field Value","description":"Write one non-list field value on a company or person. The value type must match how the field is defined.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"entityType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Which entity to write the field on: companies or persons"},"entityId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of that company or person"},"fieldId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The field ID to write"},"value":{"type":"json","required":true,"visibility":"user-or-llm","description":"The new value as {type, data}, where type matches the field\'s value type. Examples: {\\"type\\":\\"text\\",\\"data\\":\\"Series B\\"}, {\\"type\\":\\"number\\",\\"data\\":42}, {\\"type\\":\\"dropdown\\",\\"data\\":{\\"dropdownOptionId\\":7}}, {\\"type\\":\\"person\\",\\"data\\":{\\"id\\":123}}, {\\"type\\":\\"person-multi\\",\\"data\\":[{\\"id\\":123}]}. Pass data as null to clear the field"}},"hostedApiKey":"none"},"affinity_update_list_entry_field":{"id":"affinity_update_list_entry_field","name":"Affinity Update List Entry Field","description":"Write one field value on a list row. Requires the \\"Export data from Lists\\" permission.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"listEntryId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list entry ID"},"fieldId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The field ID to write"},"value":{"type":"json","required":true,"visibility":"user-or-llm","description":"The new value as {type, data}, where type matches the field\'s value type. Examples: {\\"type\\":\\"text\\",\\"data\\":\\"Series B\\"}, {\\"type\\":\\"number\\",\\"data\\":42}, {\\"type\\":\\"dropdown\\",\\"data\\":{\\"dropdownOptionId\\":7}}, {\\"type\\":\\"person\\",\\"data\\":{\\"id\\":123}}, {\\"type\\":\\"person-multi\\",\\"data\\":[{\\"id\\":123}]}. Pass data as null to clear the field"}},"hostedApiKey":"none"},"affinity_update_list_field_dropdown_option":{"id":"affinity_update_list_field_dropdown_option","name":"Affinity Update List Field Dropdown Option","description":"Change a dropdown option on a list field. Every field is optional — supply only what should change, and only fields the option\'s kind actually has.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"listId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The list ID"},"fieldId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The dropdown field ID on that list"},"dropdownOptionId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The dropdown option ID to update"},"text":{"type":"string","required":false,"visibility":"user-or-llm","description":"Replacement option label. Supply at least one field to change"},"rank":{"type":"number","required":false,"visibility":"user-or-llm","description":"Sort order. Required on a ranked-dropdown or status-dropdown option"},"color":{"type":"string","required":false,"visibility":"user-or-llm","description":"Option color: white, gray, blue, green, purple, orange, or red. Required on a ranked-dropdown or status-dropdown option"},"statusCategory":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pipeline meaning of the option: open, won, lost, or on-hold. Status-dropdown options only"},"winRate":{"type":"number","required":false,"visibility":"user-or-llm","description":"Expected win rate of the status. Status-dropdown options only"}},"hostedApiKey":"none"},"affinity_update_note":{"id":"affinity_update_note","name":"Affinity Update Note","description":"Rewrite a note\'s body or replace which records it is attached to. Each list of IDs replaces that association wholesale, an empty list clears it, and omitting one leaves it untouched. A note\'s type cannot be changed.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Affinity API key, sent as a bearer token"},"noteId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The note ID to update"},"html":{"type":"string","required":false,"visibility":"user-or-llm","description":"Replacement note body as HTML"},"companyIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Replacement set of attached companies, e.g. [1, 2]. Send [] to detach every company; omit to leave them unchanged"},"personIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Replacement set of attached persons, e.g. [1, 2]. Send [] to detach every person; omit to leave them unchanged"},"opportunityIds":{"type":"json","required":false,"visibility":"user-or-llm","description":"Replacement set of attached opportunities, e.g. [1, 2]. Send [] to detach every opportunity; omit to leave them unchanged"}},"hostedApiKey":"none"},"agentmail_create_draft":{"id":"agentmail_create_draft","name":"Create Draft","description":"Create a new email draft in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to create the draft in"},"to":{"type":"string","required":false,"visibility":"user-or-llm","description":"Recipient email addresses (comma-separated)"},"subject":{"type":"string","required":false,"visibility":"user-or-llm","description":"Draft subject line"},"text":{"type":"string","required":false,"visibility":"user-or-llm","description":"Plain text draft body"},"html":{"type":"string","required":false,"visibility":"user-or-llm","description":"HTML draft body"},"cc":{"type":"string","required":false,"visibility":"user-or-llm","description":"CC recipient email addresses (comma-separated)"},"bcc":{"type":"string","required":false,"visibility":"user-or-llm","description":"BCC recipient email addresses (comma-separated)"},"inReplyTo":{"type":"string","required":false,"visibility":"user-or-llm","description":"ID of message being replied to"},"sendAt":{"type":"string","required":false,"visibility":"user-or-llm","description":"ISO 8601 timestamp to schedule sending"}},"hostedApiKey":"none"},"agentmail_create_inbox":{"id":"agentmail_create_inbox","name":"Create Inbox","description":"Create a new email inbox with AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"username":{"type":"string","required":false,"visibility":"user-or-llm","description":"Username for the inbox email address"},"domain":{"type":"string","required":false,"visibility":"user-or-llm","description":"Domain for the inbox email address"},"displayName":{"type":"string","required":false,"visibility":"user-or-llm","description":"Display name for the inbox"}},"hostedApiKey":"none"},"agentmail_delete_draft":{"id":"agentmail_delete_draft","name":"Delete Draft","description":"Delete an email draft in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the draft"},"draftId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the draft to delete"}},"hostedApiKey":"none"},"agentmail_delete_inbox":{"id":"agentmail_delete_inbox","name":"Delete Inbox","description":"Delete an email inbox in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to delete"}},"hostedApiKey":"none"},"agentmail_delete_thread":{"id":"agentmail_delete_thread","name":"Delete Thread","description":"Delete an email thread in AgentMail (moves to trash, or permanently deletes if already in trash)","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the thread"},"threadId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the thread to delete"},"permanent":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Force permanent deletion instead of moving to trash"}},"hostedApiKey":"none"},"agentmail_forward_message":{"id":"agentmail_forward_message","name":"Forward Message","description":"Forward an email message to new recipients in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the message"},"messageId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the message to forward"},"to":{"type":"string","required":true,"visibility":"user-or-llm","description":"Recipient email addresses (comma-separated)"},"subject":{"type":"string","required":false,"visibility":"user-or-llm","description":"Override subject line"},"text":{"type":"string","required":false,"visibility":"user-or-llm","description":"Additional plain text to prepend"},"html":{"type":"string","required":false,"visibility":"user-or-llm","description":"Additional HTML to prepend"},"cc":{"type":"string","required":false,"visibility":"user-or-llm","description":"CC recipient email addresses (comma-separated)"},"bcc":{"type":"string","required":false,"visibility":"user-or-llm","description":"BCC recipient email addresses (comma-separated)"}},"hostedApiKey":"none"},"agentmail_get_draft":{"id":"agentmail_get_draft","name":"Get Draft","description":"Get details of a specific email draft in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox the draft belongs to"},"draftId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the draft to retrieve"}},"hostedApiKey":"none"},"agentmail_get_inbox":{"id":"agentmail_get_inbox","name":"Get Inbox","description":"Get details of a specific email inbox in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to retrieve"}},"hostedApiKey":"none"},"agentmail_get_message":{"id":"agentmail_get_message","name":"Get Message","description":"Get details of a specific email message in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the message"},"messageId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the message to retrieve"}},"hostedApiKey":"none"},"agentmail_get_thread":{"id":"agentmail_get_thread","name":"Get Thread","description":"Get details of a specific email thread including messages in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the thread"},"threadId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the thread to retrieve"}},"hostedApiKey":"none"},"agentmail_list_drafts":{"id":"agentmail_list_drafts","name":"List Drafts","description":"List email drafts in an inbox in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to list drafts from"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of drafts to return"},"pageToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token for next page of results"}},"hostedApiKey":"none"},"agentmail_list_inboxes":{"id":"agentmail_list_inboxes","name":"List Inboxes","description":"List all email inboxes in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of inboxes to return"},"pageToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token for next page of results"}},"hostedApiKey":"none"},"agentmail_list_messages":{"id":"agentmail_list_messages","name":"List Messages","description":"List messages in an inbox in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to list messages from"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of messages to return"},"pageToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token for next page of results"}},"hostedApiKey":"none"},"agentmail_list_threads":{"id":"agentmail_list_threads","name":"List Threads","description":"List email threads in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to list threads from"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of threads to return"},"pageToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token for next page of results"},"labels":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated labels to filter threads by"},"before":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter threads before this ISO 8601 timestamp"},"after":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter threads after this ISO 8601 timestamp"}},"hostedApiKey":"none"},"agentmail_reply_message":{"id":"agentmail_reply_message","name":"Reply to Message","description":"Reply to an existing email message in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to reply from"},"messageId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the message to reply to"},"text":{"type":"string","required":false,"visibility":"user-or-llm","description":"Plain text reply body"},"html":{"type":"string","required":false,"visibility":"user-or-llm","description":"HTML reply body"},"to":{"type":"string","required":false,"visibility":"user-or-llm","description":"Override recipient email addresses (comma-separated)"},"cc":{"type":"string","required":false,"visibility":"user-or-llm","description":"CC email addresses (comma-separated)"},"bcc":{"type":"string","required":false,"visibility":"user-or-llm","description":"BCC email addresses (comma-separated)"},"replyAll":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Reply to all recipients of the original message"}},"hostedApiKey":"none"},"agentmail_send_draft":{"id":"agentmail_send_draft","name":"Send Draft","description":"Send an existing email draft in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the draft"},"draftId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the draft to send"}},"hostedApiKey":"none"},"agentmail_send_message":{"id":"agentmail_send_message","name":"Send Message","description":"Send an email message from an AgentMail inbox","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to send from"},"to":{"type":"string","required":true,"visibility":"user-or-llm","description":"Recipient email address (comma-separated for multiple)"},"subject":{"type":"string","required":true,"visibility":"user-or-llm","description":"Email subject line"},"text":{"type":"string","required":false,"visibility":"user-or-llm","description":"Plain text email body"},"html":{"type":"string","required":false,"visibility":"user-or-llm","description":"HTML email body"},"cc":{"type":"string","required":false,"visibility":"user-or-llm","description":"CC recipient email addresses (comma-separated)"},"bcc":{"type":"string","required":false,"visibility":"user-or-llm","description":"BCC recipient email addresses (comma-separated)"}},"hostedApiKey":"none"},"agentmail_update_draft":{"id":"agentmail_update_draft","name":"Update Draft","description":"Update an existing email draft in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the draft"},"draftId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the draft to update"},"to":{"type":"string","required":false,"visibility":"user-or-llm","description":"Recipient email addresses (comma-separated)"},"subject":{"type":"string","required":false,"visibility":"user-or-llm","description":"Draft subject line"},"text":{"type":"string","required":false,"visibility":"user-or-llm","description":"Plain text draft body"},"html":{"type":"string","required":false,"visibility":"user-or-llm","description":"HTML draft body"},"cc":{"type":"string","required":false,"visibility":"user-or-llm","description":"CC recipient email addresses (comma-separated)"},"bcc":{"type":"string","required":false,"visibility":"user-or-llm","description":"BCC recipient email addresses (comma-separated)"},"sendAt":{"type":"string","required":false,"visibility":"user-or-llm","description":"ISO 8601 timestamp to schedule sending"}},"hostedApiKey":"none"},"agentmail_update_inbox":{"id":"agentmail_update_inbox","name":"Update Inbox","description":"Update the display name of an email inbox in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox to update"},"displayName":{"type":"string","required":true,"visibility":"user-or-llm","description":"New display name for the inbox"}},"hostedApiKey":"none"},"agentmail_update_message":{"id":"agentmail_update_message","name":"Update Message","description":"Add or remove labels on an email message in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the message"},"messageId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the message to update"},"addLabels":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated labels to add to the message"},"removeLabels":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated labels to remove from the message"}},"hostedApiKey":"none"},"agentmail_update_thread":{"id":"agentmail_update_thread","name":"Update Thread Labels","description":"Add or remove labels on an email thread in AgentMail","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentMail API key"},"inboxId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the inbox containing the thread"},"threadId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the thread to update"},"addLabels":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated labels to add to the thread"},"removeLabels":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated labels to remove from the thread"}},"hostedApiKey":"none"},"agentphone_create_call":{"id":"agentphone_create_call","name":"Create Outbound Call","description":"Initiate an outbound voice call from an AgentPhone agent","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"agentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Agent that will handle the call"},"toNumber":{"type":"string","required":true,"visibility":"user-or-llm","description":"Phone number to call in E.164 format (e.g. +14155551234)"},"fromNumberId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Phone number ID to use as caller ID. Must belong to the agent. If omitted, the agent\'s first assigned number is used."},"initialGreeting":{"type":"string","required":false,"visibility":"user-or-llm","description":"Optional greeting spoken when the recipient answers"},"voice":{"type":"string","required":false,"visibility":"user-or-llm","description":"Voice ID override for this call (defaults to the agent\'s configured voice)"},"systemPrompt":{"type":"string","required":false,"visibility":"user-or-llm","description":"When provided, uses a built-in LLM for the conversation instead of forwarding to your webhook"}},"hostedApiKey":"none"},"agentphone_create_contact":{"id":"agentphone_create_contact","name":"Create Contact","description":"Create a new contact in AgentPhone","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"phoneNumber":{"type":"string","required":true,"visibility":"user-or-llm","description":"Phone number in E.164 format (e.g. +14155551234)"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Contact\'s full name"},"email":{"type":"string","required":false,"visibility":"user-or-llm","description":"Contact\'s email address"},"notes":{"type":"string","required":false,"visibility":"user-or-llm","description":"Freeform notes stored on the contact"}},"hostedApiKey":"none"},"agentphone_create_number":{"id":"agentphone_create_number","name":"Create Phone Number","description":"Provision a new SMS- and voice-enabled phone number","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Two-letter country code (e.g. US, CA). Defaults to US."},"areaCode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Preferred area code (US/CA only, e.g. \\"415\\"). Best-effort — may be ignored if unavailable."},"agentId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Optionally attach the number to an agent immediately"}},"hostedApiKey":"none"},"agentphone_delete_contact":{"id":"agentphone_delete_contact","name":"Delete Contact","description":"Delete a contact by ID","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"contactId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Contact ID"}},"hostedApiKey":"none"},"agentphone_get_call":{"id":"agentphone_get_call","name":"Get Call","description":"Fetch a call and its full transcript","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"callId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the call to retrieve"}},"hostedApiKey":"none"},"agentphone_get_call_transcript":{"id":"agentphone_get_call_transcript","name":"Get Call Transcript","description":"Get the full ordered transcript for a call","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"callId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the call to retrieve the transcript for"}},"hostedApiKey":"none"},"agentphone_get_contact":{"id":"agentphone_get_contact","name":"Get Contact","description":"Fetch a single contact by ID","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"contactId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Contact ID"}},"hostedApiKey":"none"},"agentphone_get_conversation":{"id":"agentphone_get_conversation","name":"Get Conversation","description":"Get a conversation along with its recent messages","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"conversationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Conversation ID"},"messageLimit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of recent messages to include (default 50, max 100)"}},"hostedApiKey":"none"},"agentphone_get_conversation_messages":{"id":"agentphone_get_conversation_messages","name":"Get Conversation Messages","description":"Get paginated messages for a conversation","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"conversationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Conversation ID"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of messages to return (default 50, max 200)"},"before":{"type":"string","required":false,"visibility":"user-or-llm","description":"Return messages received before this ISO 8601 timestamp"},"after":{"type":"string","required":false,"visibility":"user-or-llm","description":"Return messages received after this ISO 8601 timestamp"}},"hostedApiKey":"none"},"agentphone_get_number_messages":{"id":"agentphone_get_number_messages","name":"Get Phone Number Messages","description":"Fetch messages received on a specific phone number","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"numberId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the phone number"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of messages to return (default 50, max 200)"},"before":{"type":"string","required":false,"visibility":"user-or-llm","description":"Return messages received before this ISO 8601 timestamp"},"after":{"type":"string","required":false,"visibility":"user-or-llm","description":"Return messages received after this ISO 8601 timestamp"}},"hostedApiKey":"none"},"agentphone_get_usage":{"id":"agentphone_get_usage","name":"Get Usage","description":"Retrieve current usage statistics for the AgentPhone account","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"}},"hostedApiKey":"none"},"agentphone_get_usage_daily":{"id":"agentphone_get_usage_daily","name":"Get Daily Usage","description":"Get a daily breakdown of usage (messages, calls, webhooks) for the last N days","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"days":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of days to return (1-365, default 30)"}},"hostedApiKey":"none"},"agentphone_get_usage_monthly":{"id":"agentphone_get_usage_monthly","name":"Get Monthly Usage","description":"Get monthly usage aggregation (messages, calls, webhooks) for the last N months","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"months":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of months to return (1-24, default 6)"}},"hostedApiKey":"none"},"agentphone_list_calls":{"id":"agentphone_list_calls","name":"List Calls","description":"List voice calls for this AgentPhone account","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to return (default 20, max 100)"},"offset":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to skip (min 0)"},"status":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter by status (completed, in-progress, failed)"},"direction":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter by direction (inbound, outbound)"},"type":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter by call type (pstn, web)"},"search":{"type":"string","required":false,"visibility":"user-or-llm","description":"Search by phone number (matches fromNumber or toNumber)"}},"hostedApiKey":"none"},"agentphone_list_contacts":{"id":"agentphone_list_contacts","name":"List Contacts","description":"List contacts for this AgentPhone account","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"search":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter by name or phone number (case-insensitive contains)"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to return (default 50, max 200)"},"offset":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to skip (min 0)"}},"hostedApiKey":"none"},"agentphone_list_conversations":{"id":"agentphone_list_conversations","name":"List Conversations","description":"List conversations (message threads) for this AgentPhone account","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to return (default 20, max 100)"},"offset":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to skip (min 0)"}},"hostedApiKey":"none"},"agentphone_list_numbers":{"id":"agentphone_list_numbers","name":"List Phone Numbers","description":"List all phone numbers provisioned for this AgentPhone account","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to return (default 20, max 100)"},"offset":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to skip (min 0)"}},"hostedApiKey":"none"},"agentphone_react_to_message":{"id":"agentphone_react_to_message","name":"React to Message","description":"Send an iMessage tapback reaction to a message (iMessage only)","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"messageId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the message to react to"},"reaction":{"type":"string","required":true,"visibility":"user-or-llm","description":"Reaction type: love, like, dislike, laugh, emphasize, or question"}},"hostedApiKey":"none"},"agentphone_release_number":{"id":"agentphone_release_number","name":"Release Phone Number","description":"Release (delete) a phone number. This action is irreversible.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"numberId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the phone number to release"}},"hostedApiKey":"none"},"agentphone_send_message":{"id":"agentphone_send_message","name":"Send Message","description":"Send an outbound SMS or iMessage from an AgentPhone agent","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"agentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Agent sending the message"},"toNumber":{"type":"string","required":true,"visibility":"user-or-llm","description":"Recipient phone number in E.164 format (e.g. +14155551234)"},"body":{"type":"string","required":true,"visibility":"user-or-llm","description":"Message text to send"},"mediaUrl":{"type":"string","required":false,"visibility":"user-or-llm","description":"Optional URL of an image, video, or file to attach"},"numberId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Phone number ID to send from. If omitted, the agent\'s first assigned number is used."}},"hostedApiKey":"none"},"agentphone_update_contact":{"id":"agentphone_update_contact","name":"Update Contact","description":"Update a contact\'s fields","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"contactId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Contact ID"},"phoneNumber":{"type":"string","required":false,"visibility":"user-or-llm","description":"New phone number in E.164 format"},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"New contact name"},"email":{"type":"string","required":false,"visibility":"user-or-llm","description":"New email address"},"notes":{"type":"string","required":false,"visibility":"user-or-llm","description":"New freeform notes"}},"hostedApiKey":"none"},"agentphone_update_conversation":{"id":"agentphone_update_conversation","name":"Update Conversation","description":"Update conversation metadata (stored state). Pass null to clear existing metadata.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"AgentPhone API key"},"conversationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Conversation ID"},"metadata":{"type":"json","required":false,"visibility":"user-or-llm","description":"Custom key-value metadata to store on the conversation. Pass null to clear existing metadata."}},"hostedApiKey":"none"},"agiloft_async_status":{"id":"agiloft_async_status","name":"Agiloft Async Status","description":"Check whether an asynchronous Agiloft call, such as a run action button, has completed.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table the asynchronous call was made against"},"callbackId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Callback ID returned by the asynchronous call, e.g. from Run Action Button"}},"hostedApiKey":"none"},"agiloft_attach_file":{"id":"agiloft_attach_file","name":"Agiloft Attach File","description":"Attach a file to a field in an Agiloft record.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record to attach the file to"},"fieldName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the attachment field"},"file":{"type":"file","required":true,"visibility":"user-or-llm","description":"File to attach"},"fileName":{"type":"string","required":false,"visibility":"user-or-llm","description":"Name to assign to the file (defaults to original file name)"},"overwrite":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Replace the contents of the field instead of adding another file to it"}},"hostedApiKey":"none"},"agiloft_attachment_info":{"id":"agiloft_attachment_info","name":"Agiloft Attachment Info","description":"Get information about file attachments on a record field.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record to check attachments on"},"fieldName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the attachment field to inspect"}},"hostedApiKey":"none"},"agiloft_create_record":{"id":"agiloft_create_record","name":"Agiloft Create Record","description":"Create a new record in an Agiloft table.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\", \\"contacts.employees\\")"},"data":{"type":"string","required":true,"visibility":"user-or-llm","description":"Record field values as a JSON object (e.g., {\\"first_name\\": \\"John\\", \\"status\\": \\"Active\\"})"}},"hostedApiKey":"none"},"agiloft_delete_record":{"id":"agiloft_delete_record","name":"Agiloft Delete Record","description":"Delete a record from an Agiloft table.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\", \\"contacts.employees\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record to delete"},"substituteIds":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated IDs of records that adopt the dependants of the deleted record. Read only when the delete rule is REPLACE_WITH_ANOTHER."},"deleteRule":{"type":"string","required":false,"visibility":"user-or-llm","description":"How to treat records that depend on this one: ERROR_IF_DEPENDANTS (default — fails rather than cascading), APPLY_DELETE_WHERE_POSSIBLE, DELETE_WHERE_POSSIBLE_OTHERWISE_UNLINK, APPLY_UNLINK, UNLINK_WHERE_POSSIBLE_OTHERWISE_DELETE, or REPLACE_WITH_ANOTHER"}},"hostedApiKey":"none"},"agiloft_get_choice_line_id":{"id":"agiloft_get_choice_line_id","name":"Agiloft Get Choice Line ID","description":"Resolve the internal numeric ID of a choice-list value, for use in EWSelect WHERE clauses against choice fields.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"case\\", \\"contracts\\")"},"fieldName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Choice field name (e.g., \\"priority\\", \\"status\\")"},"value":{"type":"string","required":true,"visibility":"user-or-llm","description":"Choice display value to resolve (e.g., \\"High\\", \\"Active\\")"}},"hostedApiKey":"none"},"agiloft_list_tables":{"id":"agiloft_list_tables","name":"Agiloft List Tables","description":"List the tables and fields in an Agiloft knowledge base, to discover the logical names other operations need.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":false,"visibility":"user-or-llm","description":"Logical name of a single table to describe (e.g., \\"contacts\\"). Leave empty to list every table in the knowledge base."},"includeLinkedInfo":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Include the source table and column behind each linked field"},"skipColumnsInfo":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Return table names only, omitting field details, for a much smaller response"}},"hostedApiKey":"none"},"agiloft_lock_record":{"id":"agiloft_lock_record","name":"Agiloft Lock Record","description":"Lock, unlock, or check the lock status of an Agiloft record.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record to lock, unlock, or check"},"lockAction":{"type":"string","required":true,"visibility":"user-or-llm","description":"Action to perform: \\"lock\\", \\"unlock\\", or \\"check\\""},"force":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Unlock only: release a lock held by another user."}},"hostedApiKey":"none"},"agiloft_nlp_search":{"id":"agiloft_nlp_search","name":"Agiloft Natural Language Search","description":"Search Agiloft records by describing what you want in plain language, such as \\"active NDAs submitted last month\\".","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"nlpQuery":{"type":"string","required":true,"visibility":"user-or-llm","description":"The request in plain language, e.g. \\"Show me open, high-priority contracts\\". Structured field filters are not accepted — use Search Records for those."},"fields":{"type":"string","required":true,"visibility":"user-or-llm","description":"Comma-separated field names to return, e.g. \\"id, contract_title1, company_name\\""},"page":{"type":"string","required":false,"visibility":"user-or-llm","description":"Page number, starting from 0"},"limit":{"type":"string","required":false,"visibility":"user-or-llm","description":"Records per page"}},"hostedApiKey":"none"},"agiloft_read_record":{"id":"agiloft_read_record","name":"Agiloft Read Record","description":"Read a record by ID from an Agiloft table.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\", \\"contacts.employees\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record to read"},"fields":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated list of field names to include in the response"}},"hostedApiKey":"none"},"agiloft_remove_attachment":{"id":"agiloft_remove_attachment","name":"Agiloft Remove Attachment","description":"Remove an attached file from a field in an Agiloft record.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record containing the attachment"},"fieldName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the attachment field"},"position":{"type":"string","required":true,"visibility":"user-or-llm","description":"Position index of the file to remove (starting from 0)"}},"hostedApiKey":"none"},"agiloft_retrieve_attachment":{"id":"agiloft_retrieve_attachment","name":"Agiloft Retrieve Attachment","description":"Download an attached file from an Agiloft record field.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record containing the attachment"},"fieldName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the attachment field"},"position":{"type":"string","required":true,"visibility":"user-or-llm","description":"Position index of the file in the field (starting from 0)"}},"hostedApiKey":"none"},"agiloft_run_action_button":{"id":"agiloft_run_action_button","name":"Agiloft Run Action Button","description":"Run an action button on an Agiloft record, such as an approval or send-for-signature step.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\", \\"case\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record to run the action button on"},"actionButtonField":{"type":"string","required":true,"visibility":"user-or-llm","description":"Logical name of the field holding the action button (e.g., \\"ab_field\\")"}},"hostedApiKey":"none"},"agiloft_saved_search":{"id":"agiloft_saved_search","name":"Agiloft Saved Search","description":"List the saved searches defined for an Agiloft table.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Logical table name to list saved searches for (e.g., \\"contract\\")"}},"hostedApiKey":"none"},"agiloft_search_records":{"id":"agiloft_search_records","name":"Agiloft Search Records","description":"Search for records in an Agiloft table using a query.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name to search in (e.g., \\"contracts\\", \\"contacts.employees\\")"},"query":{"type":"string","required":false,"visibility":"user-or-llm","description":"Ad hoc EWSearch query. Combine conditions with && (and) or || (or) and quote every value — e.g. \\"summary~=\'test\'&&priority=\'High\'\\". Required unless a saved search is given."},"search":{"type":"string","required":false,"visibility":"user-or-llm","description":"Label of a saved search defined on the table (e.g., \\"C: Status is Closed\\"). Can be combined with a query to narrow it further."},"fields":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated list of field names to include in the results"},"page":{"type":"string","required":false,"visibility":"user-or-llm","description":"Page number for paginated results (starting from 0)"},"limit":{"type":"string","required":false,"visibility":"user-or-llm","description":"Maximum number of records to return per page. Agiloft treats 0 as \\"all records\\", so leave it unset or use a positive value to keep result sizes bounded."}},"hostedApiKey":"none"},"agiloft_select_records":{"id":"agiloft_select_records","name":"Agiloft Select Records","description":"Select record IDs matching a SQL WHERE clause from an Agiloft table.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\", \\"contacts.employees\\")"},"where":{"type":"string","required":true,"visibility":"user-or-llm","description":"SQL WHERE clause using database column names (e.g., \\"summary like \'%new%\'\\" or \\"assigned_person=\'John Doe\'\\"). EWSelect has no page size and returns every matching ID, so append a database limit such as \\"limit 0,200\\" to bound the result."}},"hostedApiKey":"none"},"agiloft_update_record":{"id":"agiloft_update_record","name":"Agiloft Update Record","description":"Update an existing record in an Agiloft table.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\", \\"contacts.employees\\")"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the record to update"},"data":{"type":"string","required":true,"visibility":"user-or-llm","description":"Updated field values as a JSON object (e.g., {\\"status\\": \\"Active\\", \\"priority\\": \\"High\\"})"}},"hostedApiKey":"none"},"agiloft_upsert_record":{"id":"agiloft_upsert_record","name":"Agiloft Upsert Record","description":"Create an Agiloft record, or update it when a record already matches the given fields.","version":"1.0.0","params":{"instanceUrl":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft instance URL (e.g., https://mycompany.agiloft.com)"},"knowledgeBase":{"type":"string","required":true,"visibility":"user-only","description":"Knowledge base name"},"login":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft username"},"password":{"type":"string","required":true,"visibility":"user-only","description":"Agiloft password"},"table":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table name (e.g., \\"contracts\\", \\"contacts.employees\\")"},"match":{"type":"string","required":true,"visibility":"user-or-llm","description":"Field used to find an existing record (e.g., \\"ext_id\\"). Pick something that identifies a record uniquely — if more than one record matches, Agiloft writes nothing and returns a conflict."},"async":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Queue the write instead of waiting for it. Returns a callback ID instead of a record ID; pass that to Async Status to poll the result."},"data":{"type":"string","required":true,"visibility":"user-or-llm","description":"Field values as a JSON object. On create these populate the new record; on update only the supplied fields change."}},"hostedApiKey":"none"},"ahrefs_anchors":{"id":"ahrefs_anchors","name":"Ahrefs Anchors","description":"Get the anchor text distribution for a target domain or URL\'s backlinks, showing how many links and referring domains use each anchor text.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\" or \\"https://example.com/page\\""},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match)"},"history":{"type":"string","required":false,"visibility":"user-or-llm","description":"Historical scope: \\"live\\" (currently live), \\"all_time\\" (default, includes lost backlinks), or \\"since:YYYY-MM-DD\\" (backlinks found since a date)"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_backlinks":{"id":"ahrefs_backlinks","name":"Ahrefs Backlinks","description":"Get a list of backlinks pointing to a target domain or URL. Returns details about each backlink including source URL, anchor text, and domain rating.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\" or \\"https://example.com/page\\""},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match). Example: \\"domain\\""},"history":{"type":"string","required":false,"visibility":"user-or-llm","description":"Historical scope: \\"live\\" (currently live backlinks), \\"all_time\\" (default, includes lost backlinks), or \\"since:YYYY-MM-DD\\" (backlinks found since a date)."},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_backlinks_stats":{"id":"ahrefs_backlinks_stats","name":"Ahrefs Backlinks Stats","description":"Get backlink and referring domain totals for a target domain or URL, both currently live and across all time.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\" or \\"https://example.com/page\\""},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match). Example: \\"domain\\""},"date":{"type":"string","required":false,"visibility":"user-only","description":"Date to report metrics on, in YYYY-MM-DD format (defaults to today)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_batch_analysis":{"id":"ahrefs_batch_analysis","name":"Ahrefs Batch Analysis","description":"Get bulk SEO metrics (Domain Rating, backlinks, referring domains, organic traffic, and more) for multiple domains or URLs in a single request. Useful for comparing many competitors at once.","version":"1.0.0","params":{"targets":{"type":"string","required":true,"visibility":"user-or-llm","description":"Comma-separated list of domains or URLs to analyze. Example: \\"example.com,competitor.com\\""},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode applied to every target: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match)"},"protocol":{"type":"string","required":false,"visibility":"user-or-llm","description":"Protocol applied to every target: \\"both\\" (default), \\"http\\", or \\"https\\""},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for traffic data. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"volumeMode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Search volume calculation: \\"monthly\\" or \\"average\\" (default: \\"monthly\\")"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_broken_backlinks":{"id":"ahrefs_broken_backlinks","name":"Ahrefs Broken Backlinks","description":"Get a list of broken backlinks pointing to a target domain or URL. Useful for identifying link reclamation opportunities.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\" or \\"https://example.com/page\\""},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match). Example: \\"domain\\""},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_domain_rating":{"id":"ahrefs_domain_rating","name":"Ahrefs Domain Rating","description":"Get the Domain Rating (DR) and Ahrefs Rank for a target domain. Domain Rating shows the strength of a website\'s backlink profile on a scale from 0 to 100.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain to analyze (e.g., example.com)"},"date":{"type":"string","required":false,"visibility":"user-only","description":"Date for historical data in YYYY-MM-DD format (defaults to today)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_domain_rating_history":{"id":"ahrefs_domain_rating_history","name":"Ahrefs Domain Rating History","description":"Get the historical Domain Rating (DR) trend for a target domain or URL over a date range, grouped daily, weekly, or monthly.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\""},"dateFrom":{"type":"string","required":true,"visibility":"user-only","description":"Start date of the historical period, in YYYY-MM-DD format"},"dateTo":{"type":"string","required":false,"visibility":"user-only","description":"End date of the historical period, in YYYY-MM-DD format (defaults to today)"},"historyGrouping":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval for grouping data points: \\"daily\\", \\"weekly\\", or \\"monthly\\" (default: \\"monthly\\")"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_keyword_overview":{"id":"ahrefs_keyword_overview","name":"Ahrefs Keyword Overview","description":"Get detailed metrics for a keyword including search volume, keyword difficulty, CPC, clicks, and traffic potential.","version":"1.0.0","params":{"keyword":{"type":"string","required":true,"visibility":"user-or-llm","description":"The keyword to analyze"},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for keyword data. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_keywords_history":{"id":"ahrefs_keywords_history","name":"Ahrefs Keywords History","description":"Get the historical organic keyword ranking distribution for a target domain or URL over a date range: how many keywords rank in each position bucket at each point in time.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\""},"dateFrom":{"type":"string","required":true,"visibility":"user-only","description":"Start date of the historical period, in YYYY-MM-DD format"},"dateTo":{"type":"string","required":false,"visibility":"user-only","description":"End date of the historical period, in YYYY-MM-DD format (defaults to today)"},"historyGrouping":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval for grouping data points: \\"daily\\", \\"weekly\\", or \\"monthly\\" (default: \\"monthly\\")"},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for search results. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_metrics":{"id":"ahrefs_metrics","name":"Ahrefs Metrics","description":"Get a one-call organic and paid search overview for a target domain or URL: organic traffic, organic keywords, paid traffic, paid keywords, and estimated traffic cost.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\""},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for traffic data. Example: \\"us\\", \\"gb\\", \\"de\\""},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match). Example: \\"domain\\""},"date":{"type":"string","required":false,"visibility":"user-only","description":"Date to report metrics on, in YYYY-MM-DD format (defaults to today)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_metrics_history":{"id":"ahrefs_metrics_history","name":"Ahrefs Metrics History","description":"Get the historical organic and paid traffic trend for a target domain or URL over a date range: organic traffic/cost and paid traffic/cost at each point in time.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\""},"dateFrom":{"type":"string","required":true,"visibility":"user-only","description":"Start date of the historical period, in YYYY-MM-DD format"},"dateTo":{"type":"string","required":false,"visibility":"user-only","description":"End date of the historical period, in YYYY-MM-DD format (defaults to today)"},"volumeMode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Search volume calculation: \\"monthly\\" or \\"average\\" (default: \\"monthly\\")"},"historyGrouping":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval for grouping data points: \\"daily\\", \\"weekly\\", or \\"monthly\\" (default: \\"monthly\\")"},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for traffic data. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_organic_competitors":{"id":"ahrefs_organic_competitors","name":"Ahrefs Organic Competitors","description":"Get domains that compete with a target domain or URL for the same organic keywords, ranked by keyword overlap.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\""},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for search results. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match). Example: \\"domain\\""},"date":{"type":"string","required":false,"visibility":"user-only","description":"Date to report metrics on, in YYYY-MM-DD format (defaults to today)"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_organic_keywords":{"id":"ahrefs_organic_keywords","name":"Ahrefs Organic Keywords","description":"Get organic keywords that a target domain or URL ranks for in Google search results. Returns keyword details including search volume, ranking position, and estimated traffic.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\" or \\"https://example.com/page\\""},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for search results. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match). Example: \\"domain\\""},"date":{"type":"string","required":false,"visibility":"user-only","description":"Date to report metrics on, in YYYY-MM-DD format (defaults to today)"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_paid_pages":{"id":"ahrefs_paid_pages","name":"Ahrefs Paid Pages","description":"Get a target domain\'s pages that receive paid search traffic, sorted by estimated paid traffic. Returns page URLs with their paid traffic, keyword counts, and estimated spend.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\""},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for traffic data. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match)"},"date":{"type":"string","required":false,"visibility":"user-only","description":"Date to report metrics on, in YYYY-MM-DD format (defaults to today)"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_rank_tracker_competitors_overview":{"id":"ahrefs_rank_tracker_competitors_overview","name":"Ahrefs Rank Tracker Competitors Overview","description":"Get competitor rankings for the keywords tracked in an Ahrefs Rank Tracker project: each tracked keyword\'s volume and difficulty alongside every competitor\'s position, traffic, and traffic value. This endpoint is free and does not consume API units.","version":"1.0.0","params":{"projectId":{"type":"number","required":true,"visibility":"user-or-llm","description":"The Rank Tracker project ID (found in the project URL in Ahrefs)"},"date":{"type":"string","required":true,"visibility":"user-only","description":"Date to report rankings for, in YYYY-MM-DD format"},"device":{"type":"string","required":true,"visibility":"user-or-llm","description":"Rankings device type: \\"desktop\\" or \\"mobile\\""},"dateCompared":{"type":"string","required":false,"visibility":"user-only","description":"Comparison date in YYYY-MM-DD format, to compute position/traffic deltas"},"volumeMode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Search volume calculation: \\"monthly\\" or \\"average\\" (default: \\"monthly\\")"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_rank_tracker_competitors_stats":{"id":"ahrefs_rank_tracker_competitors_stats","name":"Ahrefs Rank Tracker Competitors Stats","description":"Get aggregate competitor stats for an Ahrefs Rank Tracker project: each competitor\'s traffic, traffic value, average position, and share of voice across all tracked keywords. This endpoint is free and does not consume API units.","version":"1.0.0","params":{"projectId":{"type":"number","required":true,"visibility":"user-or-llm","description":"The Rank Tracker project ID (found in the project URL in Ahrefs)"},"date":{"type":"string","required":true,"visibility":"user-only","description":"Date to report metrics for, in YYYY-MM-DD format"},"device":{"type":"string","required":true,"visibility":"user-or-llm","description":"Rankings device type: \\"desktop\\" or \\"mobile\\""},"volumeMode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Search volume calculation: \\"monthly\\" or \\"average\\" (default: \\"monthly\\")"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_rank_tracker_overview":{"id":"ahrefs_rank_tracker_overview","name":"Ahrefs Rank Tracker Overview","description":"Get ranking overview metrics for the keywords tracked in an Ahrefs Rank Tracker project: position, search volume, keyword difficulty, and estimated traffic. This endpoint is free and does not consume API units.","version":"1.0.0","params":{"projectId":{"type":"number","required":true,"visibility":"user-or-llm","description":"The Rank Tracker project ID (found in the project URL in Ahrefs)"},"date":{"type":"string","required":true,"visibility":"user-only","description":"Date to report rankings for, in YYYY-MM-DD format"},"device":{"type":"string","required":true,"visibility":"user-or-llm","description":"Rankings device type: \\"desktop\\" or \\"mobile\\""},"dateCompared":{"type":"string","required":false,"visibility":"user-only","description":"Comparison date in YYYY-MM-DD format, to compute position/traffic deltas"},"volumeMode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Search volume calculation: \\"monthly\\" or \\"average\\" (default: \\"monthly\\")"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_rank_tracker_serp_overview":{"id":"ahrefs_rank_tracker_serp_overview","name":"Ahrefs Rank Tracker SERP Overview","description":"Get the full SERP (search engine results page) for a keyword tracked in an Ahrefs Rank Tracker project, including every ranking URL with its position, title, and authority metrics. This endpoint is free and does not consume API units.","version":"1.0.0","params":{"projectId":{"type":"number","required":true,"visibility":"user-or-llm","description":"The Rank Tracker project ID (found in the project URL in Ahrefs)"},"keyword":{"type":"string","required":true,"visibility":"user-or-llm","description":"The tracked keyword to retrieve SERP data for"},"country":{"type":"string","required":true,"visibility":"user-or-llm","description":"Country code for the tracked keyword. Example: \\"us\\", \\"gb\\", \\"de\\""},"device":{"type":"string","required":true,"visibility":"user-or-llm","description":"Rankings device type: \\"desktop\\" or \\"mobile\\""},"topPositions":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of top organic positions to return (defaults to all available)"},"date":{"type":"string","required":false,"visibility":"user-only","description":"Timestamp to return the last available SERP Overview at, in YYYY-MM-DDThh:mm:ss format"},"locationId":{"type":"number","required":false,"visibility":"user-or-llm","description":"Location ID of the tracked keyword, if tracked at a specific location"},"languageCode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Language code of the tracked keyword"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_refdomains_history":{"id":"ahrefs_refdomains_history","name":"Ahrefs Referring Domains History","description":"Get the historical referring domains trend for a target domain or URL over a date range, grouped daily, weekly, or monthly.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\""},"dateFrom":{"type":"string","required":true,"visibility":"user-only","description":"Start date of the historical period, in YYYY-MM-DD format"},"dateTo":{"type":"string","required":false,"visibility":"user-only","description":"End date of the historical period, in YYYY-MM-DD format (defaults to today)"},"historyGrouping":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval for grouping data points: \\"daily\\", \\"weekly\\", or \\"monthly\\" (default: \\"monthly\\")"},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_referring_domains":{"id":"ahrefs_referring_domains","name":"Ahrefs Referring Domains","description":"Get a list of domains that link to a target domain or URL. Returns unique referring domains with their domain rating, backlink counts, and discovery dates.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain or URL to analyze. Example: \\"example.com\\" or \\"https://example.com/page\\""},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match). Example: \\"domain\\""},"history":{"type":"string","required":false,"visibility":"user-or-llm","description":"Historical scope: \\"live\\" (currently live), \\"all_time\\" (default, includes lost domains), or \\"since:YYYY-MM-DD\\" (domains found since a date)."},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_related_terms":{"id":"ahrefs_related_terms","name":"Ahrefs Related Terms","description":"Get keyword ideas related to a seed keyword: terms the same top-ranking pages also rank for (\\"also rank for\\") or also discuss (\\"also talk about\\"), with volume, difficulty, and CPC.","version":"1.0.0","params":{"keyword":{"type":"string","required":true,"visibility":"user-or-llm","description":"The seed keyword to find related terms for"},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for keyword data. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"terms":{"type":"string","required":false,"visibility":"user-or-llm","description":"Type of related keywords to return: \\"also_rank_for\\", \\"also_talk_about\\", or \\"all\\" (default: \\"all\\")"},"viewFor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Whether to derive related terms from the top 10 or top 100 ranking pages (default: \\"top_10\\")"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_site_audit_page_explorer":{"id":"ahrefs_site_audit_page_explorer","name":"Ahrefs Site Audit Page Explorer","description":"Get crawled pages from an Ahrefs Site Audit project with health and SEO metrics: HTTP status, title, link counts, backlinks, indexability, and traffic. Optionally filter to pages affected by a specific issue.","version":"1.0.0","params":{"projectId":{"type":"number","required":true,"visibility":"user-or-llm","description":"The Site Audit project ID (found in the project URL in Ahrefs)"},"date":{"type":"string","required":false,"visibility":"user-only","description":"Crawl date in YYYY-MM-DDThh:mm:ss format (defaults to the most recent crawl)"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"offset":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of results to skip, for pagination"},"issueId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Only return pages affected by this issue ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"ahrefs_top_pages":{"id":"ahrefs_top_pages","name":"Ahrefs Top Pages","description":"Get the top pages of a target domain sorted by organic traffic. Returns page URLs with their traffic, keyword counts, and estimated traffic value.","version":"1.0.0","params":{"target":{"type":"string","required":true,"visibility":"user-or-llm","description":"The target domain to analyze. Example: \\"example.com\\""},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Country code for traffic data. Example: \\"us\\", \\"gb\\", \\"de\\" (default: \\"us\\")"},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Analysis mode: domain (entire domain), prefix (URL prefix), subdomains (include all subdomains, default), exact (exact URL match). Example: \\"domain\\""},"date":{"type":"string","required":false,"visibility":"user-only","description":"Date to report metrics on, in YYYY-MM-DD format (defaults to today)"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of results to return. Example: 50 (default: 1000)"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ahrefs API Key"}},"hostedApiKey":"none"},"airtable_create_records":{"id":"airtable_create_records","name":"Airtable Create Records","description":"Write new records to an Airtable table","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"},"tableId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table ID (starts with \\"tbl\\") or table name"},"records":{"type":"json","required":true,"visibility":"user-or-llm","description":"Array of records to create, each with a `fields` object"},"typecast":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"When true, Airtable automatically converts string values to the field type"}},"oauth":{"required":true,"provider":"airtable"},"hostedApiKey":"none"},"airtable_delete_records":{"id":"airtable_delete_records","name":"Airtable Delete Records","description":"Delete one or more records from an Airtable table by ID","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"},"tableId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table ID (starts with \\"tbl\\") or table name"},"recordIds":{"type":"json","required":true,"visibility":"user-or-llm","description":"Array of record IDs to delete (each starts with \\"rec\\", e.g., [\\"recXXXXXXXXXXXXXX\\"]). Pass a single-element array to delete one record."}},"oauth":{"required":true,"provider":"airtable"},"hostedApiKey":"none"},"airtable_get_base_schema":{"id":"airtable_get_base_schema","name":"Airtable Get Base Schema","description":"Get the schema of all tables, fields, and views in an Airtable base","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"}},"oauth":{"required":true,"provider":"airtable"},"hostedApiKey":"none"},"airtable_get_record":{"id":"airtable_get_record","name":"Airtable Get Record","description":"Retrieve a single record from an Airtable table by its ID","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"},"tableId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table ID (starts with \\"tbl\\") or table name"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Record ID to retrieve (starts with \\"rec\\", e.g., \\"recXXXXXXXXXXXXXX\\")"}},"oauth":{"required":true,"provider":"airtable"},"hostedApiKey":"none"},"airtable_list_bases":{"id":"airtable_list_bases","name":"Airtable List Bases","description":"List all bases the authenticated user has access to","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"offset":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination offset for retrieving additional bases"}},"oauth":{"required":true,"provider":"airtable"},"hostedApiKey":"none"},"airtable_list_records":{"id":"airtable_list_records","name":"Airtable List Records","description":"Read records from an Airtable table","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"},"tableId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table ID (starts with \\"tbl\\") or table name"},"maxRecords":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of records to return (default: all records)"},"filterFormula":{"type":"string","required":false,"visibility":"user-or-llm","description":"Formula to filter records (e.g., \\"({Field Name} = \'Value\')\\")"}},"oauth":{"required":true,"provider":"airtable"},"hostedApiKey":"none"},"airtable_list_tables":{"id":"airtable_list_tables","name":"Airtable List Tables","description":"List all tables and their schema in an Airtable base","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"}},"oauth":{"required":true,"provider":"airtable"},"hostedApiKey":"none"},"airtable_update_multiple_records":{"id":"airtable_update_multiple_records","name":"Airtable Update Multiple Records","description":"Update multiple existing records in an Airtable table","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"},"tableId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table ID (starts with \\"tbl\\") or table name"},"records":{"type":"json","required":true,"visibility":"user-or-llm","description":"Array of records to update, each with an `id` and a `fields` object"},"typecast":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"When true, Airtable automatically converts string values to the field type"}},"oauth":{"required":true,"provider":"airtable"},"hostedApiKey":"none"},"airtable_update_record":{"id":"airtable_update_record","name":"Airtable Update Record","description":"Update an existing record in an Airtable table by ID","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"},"tableId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table ID (starts with \\"tbl\\") or table name"},"recordId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Record ID to update (starts with \\"rec\\", e.g., \\"recXXXXXXXXXXXXXX\\")"},"fields":{"type":"json","required":true,"visibility":"user-or-llm","description":"An object containing the field names and their new values"},"typecast":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"When true, Airtable automatically converts string values to the field type"}},"oauth":{"required":true,"provider":"airtable"},"hostedApiKey":"none"},"airtable_upsert_records":{"id":"airtable_upsert_records","name":"Airtable Upsert Records","description":"Update existing records or create new ones in an Airtable table, matching on the specified merge fields","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token"},"baseId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Airtable base ID (starts with \\"app\\", e.g., \\"appXXXXXXXXXXXXXX\\")"},"tableId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Table ID (starts with \\"tbl\\") or table name"},"records":{"type":"json","required":true,"visibility":"user-or-llm","description":"Array of records to upsert, each with a `fields` object"},"fieldsToMergeOn":{"type":"json","required":true,"visibility":"user-or-llm","description":"Array of field names used to match existing records (max 3). A record is updated when all merge fields match, otherwise it is created. Example: [\\"Name\\"]"},"typecast":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"When true, Airtable automatically converts string values to the field type"}},"oauth":{"required":true,"provider":"airtable"},"hostedApiKey":"none"},"airweave_search":{"id":"airweave_search","name":"Airweave Search","description":"Search your synced data collections using Airweave. Supports semantic search with hybrid, neural, or keyword retrieval strategies. Optionally generate AI-powered answers from search results.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Airweave API Key for authentication"},"collectionId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The readable ID of the collection to search"},"query":{"type":"string","required":true,"visibility":"user-or-llm","description":"The search query text"},"limit":{"type":"number","required":false,"visibility":"user-only","description":"Maximum number of results to return (default: 100)"},"retrievalStrategy":{"type":"string","required":false,"visibility":"user-or-llm","description":"Retrieval strategy: hybrid (default), neural, or keyword"},"expandQuery":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Generate query variations to improve recall"},"rerank":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Reorder results for improved relevance using LLM"},"generateAnswer":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Generate a natural-language answer to the query"}},"hostedApiKey":"none"},"algolia_add_record":{"id":"algolia_add_record","name":"Algolia Add Record","description":"Add or replace a record in an Algolia index","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index"},"objectID":{"type":"string","required":false,"visibility":"user-or-llm","description":"Object ID for the record (auto-generated if not provided)"},"record":{"type":"json","required":true,"visibility":"user-or-llm","description":"JSON object representing the record to add"}},"hostedApiKey":"none"},"algolia_batch_operations":{"id":"algolia_batch_operations","name":"Algolia Batch Operations","description":"Perform batch add, update, partial update, or delete operations on records in an Algolia index","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index"},"requests":{"type":"json","required":true,"visibility":"user-or-llm","description":"Array of batch operations. Each item has \\"action\\" (addObject, updateObject, partialUpdateObject, partialUpdateObjectNoCreate, deleteObject, delete, clear) and \\"body\\" (the record data; must include objectID for update/delete; use an empty object {} for the index-level delete/clear actions)"}},"hostedApiKey":"none"},"algolia_browse_records":{"id":"algolia_browse_records","name":"Algolia Browse Records","description":"Browse and iterate over all records in an Algolia index using cursor pagination","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia API Key (must have browse ACL)"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index to browse"},"query":{"type":"string","required":false,"visibility":"user-or-llm","description":"Search query to filter browsed records"},"filters":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter string to narrow down results"},"attributesToRetrieve":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated list of attributes to retrieve"},"hitsPerPage":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of hits per page (default: 1000, max: 1000)"},"cursor":{"type":"string","required":false,"visibility":"user-or-llm","description":"Cursor from a previous browse response for pagination"},"aroundLatLng":{"type":"string","required":false,"visibility":"user-or-llm","description":"Coordinates for geo-search (e.g., \\"40.71,-74.01\\")"},"aroundRadius":{"type":"string","required":false,"visibility":"user-or-llm","description":"Maximum radius in meters for geo-search, or \\"all\\" for unlimited"},"insideBoundingBox":{"type":"json","required":false,"visibility":"user-or-llm","description":"Bounding box coordinates as [[lat1, lng1, lat2, lng2]] for geo-search"},"insidePolygon":{"type":"json","required":false,"visibility":"user-or-llm","description":"Polygon coordinates as [[lat1, lng1, lat2, lng2, lat3, lng3, ...]] for geo-search"}},"hostedApiKey":"none"},"algolia_clear_records":{"id":"algolia_clear_records","name":"Algolia Clear Records","description":"Clear all records from an Algolia index while keeping settings, synonyms, and rules","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key (must have deleteIndex ACL)"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index to clear"}},"hostedApiKey":"none"},"algolia_copy_move_index":{"id":"algolia_copy_move_index","name":"Algolia Copy/Move Index","description":"Copy or move an Algolia index to a new destination","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the source index"},"operation":{"type":"string","required":true,"visibility":"user-or-llm","description":"Operation to perform: \\"copy\\" or \\"move\\""},"destination":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the destination index"},"scope":{"type":"json","required":false,"visibility":"user-or-llm","description":"Array of scopes to copy (only for \\"copy\\" operation): [\\"settings\\", \\"synonyms\\", \\"rules\\"]. Omit to copy everything including records."}},"hostedApiKey":"none"},"algolia_delete_by_filter":{"id":"algolia_delete_by_filter","name":"Algolia Delete By Filter","description":"Delete all records matching a filter from an Algolia index","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key (must have deleteIndex ACL)"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index"},"filters":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter expression to match records for deletion (e.g., \\"category:outdated\\")"},"facetFilters":{"type":"json","required":false,"visibility":"user-or-llm","description":"Array of facet filters (e.g., [\\"brand:Acme\\"])"},"numericFilters":{"type":"json","required":false,"visibility":"user-or-llm","description":"Array of numeric filters (e.g., [\\"price > 100\\"])"},"tagFilters":{"type":"json","required":false,"visibility":"user-or-llm","description":"Array of tag filters using the _tags attribute (e.g., [\\"published\\"])"},"aroundLatLng":{"type":"string","required":false,"visibility":"user-or-llm","description":"Coordinates for geo-search filter (e.g., \\"40.71,-74.01\\")"},"aroundRadius":{"type":"string","required":false,"visibility":"user-or-llm","description":"Maximum radius in meters for geo-search, or \\"all\\" for unlimited"},"insideBoundingBox":{"type":"json","required":false,"visibility":"user-or-llm","description":"Bounding box coordinates as [[lat1, lng1, lat2, lng2]] for geo-search filter"},"insidePolygon":{"type":"json","required":false,"visibility":"user-or-llm","description":"Polygon coordinates as [[lat1, lng1, lat2, lng2, lat3, lng3, ...]] for geo-search filter"}},"hostedApiKey":"none"},"algolia_delete_index":{"id":"algolia_delete_index","name":"Algolia Delete Index","description":"Delete an entire Algolia index and all its records","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key (must have deleteIndex ACL)"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index to delete"}},"hostedApiKey":"none"},"algolia_delete_record":{"id":"algolia_delete_record","name":"Algolia Delete Record","description":"Delete a record by objectID from an Algolia index","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index"},"objectID":{"type":"string","required":true,"visibility":"user-or-llm","description":"The objectID of the record to delete"}},"hostedApiKey":"none"},"algolia_get_record":{"id":"algolia_get_record","name":"Algolia Get Record","description":"Get a record by objectID from an Algolia index","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index"},"objectID":{"type":"string","required":true,"visibility":"user-or-llm","description":"The objectID of the record to retrieve"},"attributesToRetrieve":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated list of attributes to retrieve"}},"hostedApiKey":"none"},"algolia_get_records":{"id":"algolia_get_records","name":"Algolia Get Records","description":"Retrieve multiple records by objectID from one or more Algolia indices","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Default index name for all requests"},"requests":{"type":"json","required":true,"visibility":"user-or-llm","description":"Array of objects specifying records to retrieve. Each must have \\"objectID\\" and optionally \\"indexName\\" and \\"attributesToRetrieve\\"."}},"hostedApiKey":"none"},"algolia_get_settings":{"id":"algolia_get_settings","name":"Algolia Get Settings","description":"Retrieve the settings of an Algolia index","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index"}},"hostedApiKey":"none"},"algolia_get_task_status":{"id":"algolia_get_task_status","name":"Algolia Get Task Status","description":"Check whether an Algolia indexing task has finished publishing","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index the task ran against"},"taskID":{"type":"number","required":true,"visibility":"user-or-llm","description":"The taskID returned by a previous write operation"}},"hostedApiKey":"none"},"algolia_list_indices":{"id":"algolia_list_indices","name":"Algolia List Indices","description":"List all indices in an Algolia application","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia API Key"},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number for paginating indices (default: not paginated)"},"hitsPerPage":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of indices per page (default: 100)"}},"hostedApiKey":"none"},"algolia_partial_update_record":{"id":"algolia_partial_update_record","name":"Algolia Partial Update Record","description":"Partially update a record in an Algolia index without replacing it entirely","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index"},"objectID":{"type":"string","required":true,"visibility":"user-or-llm","description":"The objectID of the record to update"},"attributes":{"type":"json","required":true,"visibility":"user-or-llm","description":"JSON object with attributes to update. Supports built-in operations like {\\"stock\\": {\\"_operation\\": \\"Decrement\\", \\"value\\": 1}}"},"createIfNotExists":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Whether to create the record if it does not exist (default: true)"}},"hostedApiKey":"none"},"algolia_search":{"id":"algolia_search","name":"Algolia Search","description":"Search an Algolia index","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia API Key"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index to search"},"query":{"type":"string","required":true,"visibility":"user-or-llm","description":"Search query text"},"hitsPerPage":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of hits per page (default: 20)"},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number to retrieve (default: 0)"},"filters":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter string (e.g., \\"category:electronics AND price < 100\\")"},"attributesToRetrieve":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated list of attributes to retrieve"},"facets":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated list of facet attribute names to retrieve counts for (use \\"*\\" for all)"},"getRankingInfo":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Whether to include detailed ranking information in each hit"},"aroundLatLng":{"type":"string","required":false,"visibility":"user-or-llm","description":"Coordinates for geo-search (e.g., \\"40.71,-74.01\\")"},"aroundRadius":{"type":"string","required":false,"visibility":"user-or-llm","description":"Maximum radius in meters for geo-search, or \\"all\\" for unlimited"},"insideBoundingBox":{"type":"json","required":false,"visibility":"user-or-llm","description":"Bounding box coordinates as [[lat1, lng1, lat2, lng2]] for geo-search"},"insidePolygon":{"type":"json","required":false,"visibility":"user-or-llm","description":"Polygon coordinates as [[lat1, lng1, lat2, lng2, lat3, lng3, ...]] for geo-search"}},"hostedApiKey":"none"},"algolia_update_settings":{"id":"algolia_update_settings","name":"Algolia Update Settings","description":"Update the settings of an Algolia index","version":"1.0","params":{"applicationId":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Application ID"},"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Algolia Admin API Key (must have editSettings ACL)"},"indexName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the Algolia index"},"settings":{"type":"json","required":true,"visibility":"user-or-llm","description":"JSON object with settings to update (e.g., {\\"searchableAttributes\\": [\\"name\\", \\"description\\"], \\"customRanking\\": [\\"desc(popularity)\\"]})"},"forwardToReplicas":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Whether to apply changes to replica indices (default: false)"}},"hostedApiKey":"none"},"amplitude_event_segmentation":{"id":"amplitude_event_segmentation","name":"Amplitude Event Segmentation","description":"Query event analytics data with segmentation. Get event counts, uniques, averages, and more.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"eventType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Event type name to analyze"},"start":{"type":"string","required":true,"visibility":"user-or-llm","description":"Start date in YYYYMMDD format"},"end":{"type":"string","required":true,"visibility":"user-or-llm","description":"End date in YYYYMMDD format"},"metric":{"type":"string","required":false,"visibility":"user-or-llm","description":"Metric type: uniques, totals, pct_dau, average, histogram, sums, value_avg, or formula (default: uniques)"},"interval":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval: 1 (daily), 7 (weekly), or 30 (monthly)"},"groupBy":{"type":"string","required":false,"visibility":"user-or-llm","description":"Property name to group by (prefix custom user properties with \\"gp:\\")"},"groupBy2":{"type":"string","required":false,"visibility":"user-or-llm","description":"Second property name to group by (prefix custom user properties with \\"gp:\\")"},"limit":{"type":"string","required":false,"visibility":"user-or-llm","description":"Maximum number of group-by values (max 1000)"},"filters":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON array of filter objects applied to the event, e.g. [{\\"subprop_type\\":\\"event\\",\\"subprop_key\\":\\"city\\",\\"subprop_op\\":\\"is\\",\\"subprop_value\\":[\\"San Francisco\\"]}]"},"formula":{"type":"string","required":false,"visibility":"user-or-llm","description":"Required when metric is \\"formula\\", e.g. \\"UNIQUES(A)/UNIQUES(B)\\""},"segment":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON segment definition(s) applied to the query"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}},"hostedApiKey":"none"},"amplitude_funnels":{"id":"amplitude_funnels","name":"Amplitude Funnels","description":"Analyze conversion rates and drop-off between a sequence of events.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"events":{"type":"string","required":true,"visibility":"user-or-llm","description":"JSON array of event objects, one per funnel step in order, e.g. [{\\"event_type\\":\\"signup\\"},{\\"event_type\\":\\"purchase\\"}]"},"start":{"type":"string","required":true,"visibility":"user-or-llm","description":"Start date in YYYYMMDD format"},"end":{"type":"string","required":true,"visibility":"user-or-llm","description":"End date in YYYYMMDD format"},"mode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Funnel ordering: \\"ordered\\", \\"unordered\\", or \\"sequential\\" (default: ordered)"},"userType":{"type":"string","required":false,"visibility":"user-or-llm","description":"User type: \\"new\\" or \\"active\\" (default: active)"},"interval":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval: -300000 (real-time), -3600000 (hourly), 1 (daily), 7 (weekly), or 30 (monthly)"},"conversionWindowSeconds":{"type":"string","required":false,"visibility":"user-or-llm","description":"Conversion window in seconds (default: 2592000, i.e. 30 days)"},"groupBy":{"type":"string","required":false,"visibility":"user-or-llm","description":"Property to group by (limit: one; prefix custom properties with \\"gp:\\")"},"limit":{"type":"string","required":false,"visibility":"user-or-llm","description":"Maximum number of group-by values (default: 100, max: 1000)"},"segment":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON segment definition(s) applied to the query"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}},"hostedApiKey":"none"},"amplitude_get_active_users":{"id":"amplitude_get_active_users","name":"Amplitude Get Active Users","description":"Get active or new user counts over a date range from the Dashboard REST API.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"start":{"type":"string","required":true,"visibility":"user-or-llm","description":"Start date in YYYYMMDD format"},"end":{"type":"string","required":true,"visibility":"user-or-llm","description":"End date in YYYYMMDD format"},"metric":{"type":"string","required":false,"visibility":"user-or-llm","description":"Metric type: \\"active\\" or \\"new\\" (default: active)"},"interval":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval: 1 (daily), 7 (weekly), or 30 (monthly)"},"groupBy":{"type":"string","required":false,"visibility":"user-or-llm","description":"Property name to group by"},"segment":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON segment definition(s) applied to the query"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}},"hostedApiKey":"none"},"amplitude_get_revenue":{"id":"amplitude_get_revenue","name":"Amplitude Get Revenue","description":"Get revenue LTV data including ARPU, ARPPU, total revenue, and paying user counts.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"start":{"type":"string","required":true,"visibility":"user-or-llm","description":"Start date in YYYYMMDD format"},"end":{"type":"string","required":true,"visibility":"user-or-llm","description":"End date in YYYYMMDD format"},"metric":{"type":"string","required":false,"visibility":"user-or-llm","description":"Metric: 0 (ARPU), 1 (ARPPU), 2 (Total Revenue), 3 (Paying Users)"},"interval":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval: 1 (daily), 7 (weekly), or 30 (monthly)"},"groupBy":{"type":"string","required":false,"visibility":"user-or-llm","description":"Property name to group by (limit: one)"},"segment":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON segment definition(s) applied to the query"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}},"hostedApiKey":"none"},"amplitude_group_identify":{"id":"amplitude_group_identify","name":"Amplitude Group Identify","description":"Set group-level properties in Amplitude. Supports $set, $setOnce, $add, $append, $unset operations.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"groupType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Group classification (e.g., \\"company\\", \\"org_id\\")"},"groupValue":{"type":"string","required":true,"visibility":"user-or-llm","description":"Specific group identifier (e.g., \\"Acme Corp\\")"},"groupProperties":{"type":"string","required":true,"visibility":"user-or-llm","description":"JSON object of group properties. Use operations like $set, $setOnce, $add, $append, $unset."},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}},"hostedApiKey":"none"},"amplitude_identify_user":{"id":"amplitude_identify_user","name":"Amplitude Identify User","description":"Set user properties in Amplitude using the Identify API. Supports $set, $setOnce, $add, $append, $unset operations.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"userId":{"type":"string","required":false,"visibility":"user-or-llm","description":"User ID (required if no device_id)"},"deviceId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Device ID (required if no user_id)"},"userProperties":{"type":"string","required":true,"visibility":"user-or-llm","description":"JSON object of user properties. Use operations like $set, $setOnce, $add, $append, $unset."},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}},"hostedApiKey":"none"},"amplitude_list_events":{"id":"amplitude_list_events","name":"Amplitude List Events","description":"List all event types in the Amplitude project with their weekly totals and unique counts.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}},"hostedApiKey":"none"},"amplitude_realtime_active_users":{"id":"amplitude_realtime_active_users","name":"Amplitude Real-time Active Users","description":"Get real-time active user counts at 5-minute granularity for the last 2 days.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}},"hostedApiKey":"none"},"amplitude_retention":{"id":"amplitude_retention","name":"Amplitude Retention","description":"Measure how many users return to perform an action after a starting action.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"startEvent":{"type":"string","required":true,"visibility":"user-or-llm","description":"JSON starting event object, e.g. {\\"event_type\\":\\"_new\\"} or {\\"event_type\\":\\"_active\\"}"},"returnEvent":{"type":"string","required":true,"visibility":"user-or-llm","description":"JSON returning event object, e.g. {\\"event_type\\":\\"_all\\"} or {\\"event_type\\":\\"_active\\"}"},"start":{"type":"string","required":true,"visibility":"user-or-llm","description":"Start date in YYYYMMDD format"},"end":{"type":"string","required":true,"visibility":"user-or-llm","description":"End date in YYYYMMDD format"},"retentionMode":{"type":"string","required":false,"visibility":"user-or-llm","description":"Retention type: \\"bracket\\", \\"rolling\\", or \\"n-day\\" (default: n-day)"},"retentionBrackets":{"type":"string","required":false,"visibility":"user-or-llm","description":"Required when Retention Mode is \\"bracket\\". Day ranges, e.g. [[0,4]]"},"interval":{"type":"string","required":false,"visibility":"user-or-llm","description":"Time interval: 1 (daily), 7 (weekly), or 30 (monthly)"},"groupBy":{"type":"string","required":false,"visibility":"user-or-llm","description":"Property to group by (limit: one; prefix custom properties with \\"gp:\\")"},"segment":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON segment definition(s) applied to the query"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}},"hostedApiKey":"none"},"amplitude_send_event":{"id":"amplitude_send_event","name":"Amplitude Send Event","description":"Track an event in Amplitude using the HTTP V2 API.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"userId":{"type":"string","required":false,"visibility":"user-or-llm","description":"User ID (required if no device_id)"},"deviceId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Device ID (required if no user_id)"},"eventType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the event (e.g., \\"page_view\\", \\"purchase\\")"},"eventProperties":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON object of custom event properties"},"userProperties":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON object of user properties to set (supports $set, $setOnce, $add, $append, $unset)"},"time":{"type":"string","required":false,"visibility":"user-or-llm","description":"Event timestamp in milliseconds since epoch"},"sessionId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Session start time in milliseconds since epoch"},"insertId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Unique ID for deduplication (within 7-day window)"},"appVersion":{"type":"string","required":false,"visibility":"user-or-llm","description":"Application version string"},"platform":{"type":"string","required":false,"visibility":"user-or-llm","description":"Platform (e.g., \\"Web\\", \\"iOS\\", \\"Android\\")"},"country":{"type":"string","required":false,"visibility":"user-or-llm","description":"Two-letter country code"},"language":{"type":"string","required":false,"visibility":"user-or-llm","description":"Language code (e.g., \\"en\\")"},"ip":{"type":"string","required":false,"visibility":"user-or-llm","description":"IP address for geo-location"},"price":{"type":"string","required":false,"visibility":"user-or-llm","description":"Price of the item purchased"},"quantity":{"type":"string","required":false,"visibility":"user-or-llm","description":"Quantity of items purchased"},"revenue":{"type":"string","required":false,"visibility":"user-or-llm","description":"Revenue amount"},"productId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Product identifier"},"revenueType":{"type":"string","required":false,"visibility":"user-or-llm","description":"Revenue type (e.g., \\"purchase\\", \\"refund\\")"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}},"hostedApiKey":"none"},"amplitude_user_activity":{"id":"amplitude_user_activity","name":"Amplitude User Activity","description":"Get the event stream for a specific user by their Amplitude ID.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"amplitudeId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Amplitude internal user ID"},"offset":{"type":"string","required":false,"visibility":"user-or-llm","description":"Offset for pagination (default 0)"},"limit":{"type":"string","required":false,"visibility":"user-or-llm","description":"Maximum number of events to return (default 1000, max 1000)"},"direction":{"type":"string","required":false,"visibility":"user-or-llm","description":"Sort direction: \\"latest\\" or \\"earliest\\" (default: latest)"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}},"hostedApiKey":"none"},"amplitude_user_profile":{"id":"amplitude_user_profile","name":"Amplitude User Profile","description":"Get a user profile including properties, cohort memberships, and computed properties. Not available for EU data-residency projects.","version":"1.0.0","params":{"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"userId":{"type":"string","required":false,"visibility":"user-or-llm","description":"External user ID (required if no device_id)"},"deviceId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Device ID (required if no user_id)"},"getAmpProps":{"type":"string","required":false,"visibility":"user-or-llm","description":"Include Amplitude user properties (true/false, default: false)"},"getCohortIds":{"type":"string","required":false,"visibility":"user-or-llm","description":"Include cohort IDs the user belongs to (true/false, default: false)"},"getComputations":{"type":"string","required":false,"visibility":"user-or-llm","description":"Include computed user properties (true/false, default: false)"}},"hostedApiKey":"none"},"amplitude_user_search":{"id":"amplitude_user_search","name":"Amplitude User Search","description":"Search for a user by User ID, Device ID, or Amplitude ID using the Dashboard REST API.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude API Key"},"secretKey":{"type":"string","required":true,"visibility":"user-only","description":"Amplitude Secret Key"},"user":{"type":"string","required":true,"visibility":"user-or-llm","description":"User ID, Device ID, or Amplitude ID to search for"},"dataResidency":{"type":"string","required":false,"visibility":"user-or-llm","description":"Data residency region: \\"us\\" (default) or \\"eu\\""}},"hostedApiKey":"none"},"apify_get_dataset_items":{"id":"apify_get_dataset_items","name":"APIFY Get Dataset Items","description":"Retrieve items stored in an APIFY dataset","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"APIFY API token from console.apify.com/account#/integrations"},"datasetId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Dataset ID to read items from. Example: \\"9RnD3Pql2vGZkc5H5\\""},"itemLimit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Max items to return (1-250000). Default: all items. Example: 500"},"offset":{"type":"number","required":false,"visibility":"user-or-llm","description":"Number of items to skip at the start. Default: 0"},"fields":{"type":"string","required":false,"visibility":"user-or-llm","description":"Comma-separated list of fields to include. Example: \\"title,url,price\\""}},"hostedApiKey":"none"},"apify_get_run":{"id":"apify_get_run","name":"APIFY Get Run","description":"Get the status and details of an APIFY actor run","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"APIFY API token from console.apify.com/account#/integrations"},"runId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Actor run ID to fetch. Example: \\"HG7ML7M8z78YcAPEB\\""}},"hostedApiKey":"none"},"apify_run_actor_async":{"id":"apify_run_actor_async","name":"APIFY Run Actor (Async)","description":"Run an APIFY actor asynchronously with polling for long-running tasks","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"APIFY API token from console.apify.com/account#/integrations"},"actorId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Actor ID or username/actor-name. Examples: \\"apify/web-scraper\\", \\"janedoe/my-actor\\", \\"moJRLRc85AitArpNN\\""},"input":{"type":"string","required":false,"visibility":"user-or-llm","description":"Actor input as JSON string. Example: {\\"startUrls\\": [{\\"url\\": \\"https://example.com\\"}], \\"maxPages\\": 10}"},"waitForFinish":{"type":"number","required":false,"visibility":"user-or-llm","description":"Initial wait time in seconds (0-60) before polling starts. Example: 30"},"itemLimit":{"type":"number","required":false,"default":100,"visibility":"user-or-llm","description":"Max dataset items to fetch (1-250000). Default: 100. Example: 500"},"memory":{"type":"number","required":false,"visibility":"user-or-llm","description":"Memory in megabytes allocated for the actor run (128-32768). Example: 1024 for 1GB, 2048 for 2GB"},"timeout":{"type":"number","required":false,"visibility":"user-or-llm","description":"Timeout in seconds for the actor run. Example: 300 for 5 minutes, 3600 for 1 hour"},"build":{"type":"string","required":false,"visibility":"user-or-llm","description":"Actor build to run. Examples: \\"latest\\", \\"beta\\", \\"1.2.3\\", \\"build-tag-name\\""}},"hostedApiKey":"none"},"apify_run_actor_sync":{"id":"apify_run_actor_sync","name":"APIFY Run Actor (Sync)","description":"Run an APIFY actor synchronously and get results (max 5 minutes)","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"APIFY API token from console.apify.com/account#/integrations"},"actorId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Actor ID or username/actor-name. Examples: \\"apify/web-scraper\\", \\"janedoe/my-actor\\", \\"moJRLRc85AitArpNN\\""},"input":{"type":"string","required":false,"visibility":"user-or-llm","description":"Actor input as JSON string. Example: {\\"startUrls\\": [{\\"url\\": \\"https://example.com\\"}], \\"maxPages\\": 10}"},"memory":{"type":"number","required":false,"visibility":"user-or-llm","description":"Memory in megabytes allocated for the actor run (128-32768). Example: 1024 for 1GB, 2048 for 2GB"},"timeout":{"type":"number","required":false,"visibility":"user-or-llm","description":"Timeout in seconds for the actor run. Example: 300 for 5 minutes, 3600 for 1 hour"},"build":{"type":"string","required":false,"visibility":"user-or-llm","description":"Actor build to run. Examples: \\"latest\\", \\"beta\\", \\"1.2.3\\", \\"build-tag-name\\""}},"hostedApiKey":"none"},"apify_run_task":{"id":"apify_run_task","name":"APIFY Run Task","description":"Run a saved APIFY actor task synchronously and get dataset items (max 5 minutes)","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"APIFY API token from console.apify.com/account#/integrations"},"taskId":{"type":"string","required":true,"visibility":"user-or-llm","description":"Task ID or username/task-name. Examples: \\"janedoe/my-task\\", \\"moJRLRc85AitArpNN\\""},"input":{"type":"string","required":false,"visibility":"user-or-llm","description":"JSON string that overrides the task\'s saved input. Example: {\\"startUrls\\": [{\\"url\\": \\"https://example.com\\"}]}"},"itemLimit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Max dataset items to return (1-250000). Example: 500"},"memory":{"type":"number","required":false,"visibility":"user-or-llm","description":"Memory in megabytes allocated for the run (128-32768). Example: 1024 for 1GB"},"timeout":{"type":"number","required":false,"visibility":"user-or-llm","description":"Timeout in seconds for the run. Example: 300 for 5 minutes"},"build":{"type":"string","required":false,"visibility":"user-or-llm","description":"Actor build to run. Examples: \\"latest\\", \\"beta\\", \\"1.2.3\\""}},"hostedApiKey":"none"},"apollo_account_bulk_create":{"id":"apollo_account_bulk_create","name":"Apollo Bulk Create Accounts","description":"Create up to 100 accounts at once in your Apollo database. Set run_dedupe=true to deduplicate by domain, organization_id, and name. Master key required.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"accounts":{"type":"array","required":true,"visibility":"user-or-llm","description":"Array of accounts to create (max 100). Each account should include a name, and may optionally include domain, phone, phone_status_cd, raw_address, owner_id, linkedin_url, facebook_url, twitter_url, salesforce_id, and hubspot_id."},"append_label_names":{"type":"array","required":false,"visibility":"user-only","description":"Array of label names to add to ALL accounts in this request"},"run_dedupe":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"When true, performs aggressive deduplication by domain, organization_id, and name (defaults to false)"}},"hostedApiKey":"none"},"apollo_account_bulk_update":{"id":"apollo_account_bulk_update","name":"Apollo Bulk Update Accounts","description":"Update up to 1000 existing accounts at once in your Apollo database (higher limit than contacts!). Each account must include an id field. Master key required.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"account_ids":{"type":"array","required":false,"visibility":"user-or-llm","description":"Array of account IDs to update with the same values (max 1000). Use with name/owner_id for uniform updates. Use either this OR account_attributes."},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"When using account_ids, apply this name to all accounts"},"owner_id":{"type":"string","required":false,"visibility":"user-or-llm","description":"When using account_ids, apply this owner to all accounts"},"account_stage_id":{"type":"string","required":false,"visibility":"user-or-llm","description":"When using account_ids, apply this account stage to all accounts"},"account_attributes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Array of account objects with individual updates (each must include id). Example: [{\\"id\\": \\"acc1\\", \\"name\\": \\"Acme\\", \\"owner_id\\": \\"u1\\", \\"account_stage_id\\": \\"s1\\", \\"typed_custom_fields\\": {\\"field_id\\": \\"value\\"}}]"},"async":{"type":"boolean","required":false,"visibility":"user-only","description":"When true, processes the update asynchronously. Only supported when using account_ids; returns 422 if used with account_attributes."}},"hostedApiKey":"none"},"apollo_account_create":{"id":"apollo_account_create","name":"Apollo Create Account","description":"Create a new account (company) in your Apollo database","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Company name (e.g., \\"Acme Corporation\\")"},"domain":{"type":"string","required":false,"visibility":"user-or-llm","description":"Company domain without www. prefix (e.g., \\"acme.com\\")"},"phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Primary phone number for the account"},"owner_id":{"type":"string","required":false,"visibility":"user-only","description":"Apollo user ID of the account owner"},"account_stage_id":{"type":"string","required":false,"visibility":"user-only","description":"Apollo ID for the account stage to assign this account to"},"raw_address":{"type":"string","required":false,"visibility":"user-or-llm","description":"Corporate location (e.g., \\"San Francisco, CA, USA\\")"},"typed_custom_fields":{"type":"json","required":false,"visibility":"user-only","description":"Custom field values as { custom_field_id: value } map"}},"hostedApiKey":"none"},"apollo_account_search":{"id":"apollo_account_search","name":"Apollo Search Accounts","description":"Search your team\'s accounts in Apollo. Display limit: 50,000 records (100 records per page, 500 pages max). Use filters to narrow results. Master key required.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"q_organization_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter accounts by organization name (partial-match search)"},"account_stage_ids":{"type":"array","required":false,"visibility":"user-only","description":"Filter by account stage IDs"},"account_label_ids":{"type":"array","required":false,"visibility":"user-only","description":"Filter by account label IDs"},"sort_by_field":{"type":"string","required":false,"visibility":"user-or-llm","description":"Sort field: \\"account_last_activity_date\\", \\"account_created_at\\", or \\"account_updated_at\\""},"sort_ascending":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Sort ascending when true. Defaults to descending."},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number for pagination (e.g., 1, 2, 3)"},"per_page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Results per page, max 100 (e.g., 25, 50, 100)"}},"hostedApiKey":"none"},"apollo_account_update":{"id":"apollo_account_update","name":"Apollo Update Account","description":"Update an existing account in your Apollo database","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"account_id":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the account to update (e.g., \\"acc_abc123\\")"},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Company name (e.g., \\"Acme Corporation\\")"},"domain":{"type":"string","required":false,"visibility":"user-or-llm","description":"Company domain (e.g., \\"acme.com\\")"},"phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Company phone number"},"owner_id":{"type":"string","required":false,"visibility":"user-only","description":"Apollo user ID of the account owner"},"account_stage_id":{"type":"string","required":false,"visibility":"user-only","description":"Apollo ID for the account stage to assign this account to"},"raw_address":{"type":"string","required":false,"visibility":"user-or-llm","description":"Corporate location (e.g., \\"San Francisco, CA, USA\\")"},"typed_custom_fields":{"type":"json","required":false,"visibility":"user-only","description":"Custom field values as { custom_field_id: value } map"}},"hostedApiKey":"none"},"apollo_contact_bulk_create":{"id":"apollo_contact_bulk_create","name":"Apollo Bulk Create Contacts","description":"Create up to 100 contacts at once in your Apollo database. Supports deduplication to prevent creating duplicate contacts. Master key required.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"contacts":{"type":"array","required":true,"visibility":"user-or-llm","description":"Array of contacts to create (max 100). Each contact may include first_name, last_name, email, title, organization_name, account_id, owner_id, contact_stage_id, linkedin_url, phone (single string) or phone_numbers (array of {raw_number, position}), contact_emails, typed_custom_fields, and CRM IDs (salesforce_contact_id, hubspot_id, team_id) for cross-system matching"},"append_label_names":{"type":"array","required":false,"visibility":"user-or-llm","description":"Label names to add to all contacts in this request (e.g., [\\"Hot Lead\\"])"},"run_dedupe":{"type":"boolean","required":false,"visibility":"user-only","description":"Enable deduplication to prevent creating duplicate contacts. When true, existing contacts are returned without modification"}},"hostedApiKey":"none"},"apollo_contact_bulk_update":{"id":"apollo_contact_bulk_update","name":"Apollo Bulk Update Contacts","description":"Update up to 100 existing contacts at once in your Apollo database. Each contact must include an id field. Master key required.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"contact_ids":{"type":"array","required":false,"visibility":"user-or-llm","description":"Array of contact IDs to update. Must be paired with an object-form contact_attributes specifying the fields to apply uniformly to all listed contacts."},"contact_attributes":{"type":"json","required":false,"visibility":"user-or-llm","description":"Required. Either an array of per-contact updates (each with id) — used standalone — or a single object of attributes to apply to all contact_ids. Supported fields: owner_id, email, organization_name, title, first_name, last_name, account_id, present_raw_address, linkedin_url, typed_custom_fields"},"async":{"type":"boolean","required":false,"visibility":"user-only","description":"Force asynchronous processing. Automatically enabled for >100 contacts"}},"hostedApiKey":"none"},"apollo_contact_create":{"id":"apollo_contact_create","name":"Apollo Create Contact","description":"Create a new contact in your Apollo database","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"first_name":{"type":"string","required":true,"visibility":"user-or-llm","description":"First name of the contact"},"last_name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Last name of the contact"},"email":{"type":"string","required":false,"visibility":"user-or-llm","description":"Email address of the contact"},"title":{"type":"string","required":false,"visibility":"user-or-llm","description":"Job title (e.g., \\"VP of Sales\\", \\"Software Engineer\\")"},"account_id":{"type":"string","required":false,"visibility":"user-or-llm","description":"Apollo account ID to associate with (e.g., \\"acc_abc123\\")"},"owner_id":{"type":"string","required":false,"visibility":"user-only","description":"User ID of the contact owner (accepted by Apollo but not officially documented for POST /contacts)"},"organization_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Name of the contact\'s employer (e.g., \\"Apollo\\")"},"website_url":{"type":"string","required":false,"visibility":"user-or-llm","description":"Corporate website URL (e.g., \\"https://www.apollo.io/\\")"},"label_names":{"type":"array","required":false,"visibility":"user-or-llm","description":"Lists/labels to add the contact to (e.g., [\\"Prospects\\"])"},"contact_stage_id":{"type":"string","required":false,"visibility":"user-or-llm","description":"Apollo ID for the contact stage"},"present_raw_address":{"type":"string","required":false,"visibility":"user-or-llm","description":"Personal location for the contact (e.g., \\"Atlanta, United States\\")"},"direct_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Primary phone number"},"corporate_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Work/office phone number"},"mobile_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Mobile phone number"},"home_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Home phone number"},"other_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Alternative phone number"},"typed_custom_fields":{"type":"json","required":false,"visibility":"user-or-llm","description":"Custom field values keyed by custom field ID"},"run_dedupe":{"type":"boolean","required":false,"visibility":"user-only","description":"When true, Apollo deduplicates against existing contacts"}},"hostedApiKey":"none"},"apollo_contact_search":{"id":"apollo_contact_search","name":"Apollo Search Contacts","description":"Search your team\'s contacts in Apollo","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"q_keywords":{"type":"string","required":false,"visibility":"user-or-llm","description":"Keywords to search for"},"contact_stage_ids":{"type":"array","required":false,"visibility":"user-only","description":"Filter by contact stage IDs"},"contact_label_ids":{"type":"array","required":false,"visibility":"user-only","description":"Filter by Apollo label IDs (lists)"},"sort_by_field":{"type":"string","required":false,"visibility":"user-only","description":"Sort field: contact_last_activity_date, contact_email_last_opened_at, contact_email_last_clicked_at, contact_created_at, or contact_updated_at"},"sort_ascending":{"type":"boolean","required":false,"visibility":"user-only","description":"When true, sort ascending. Must be used together with sort_by_field"},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number for pagination (e.g., 1, 2, 3)"},"per_page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Results per page, max 100 (e.g., 25, 50, 100)"}},"hostedApiKey":"none"},"apollo_contact_update":{"id":"apollo_contact_update","name":"Apollo Update Contact","description":"Update an existing contact in your Apollo database","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"contact_id":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the contact to update (e.g., \\"con_abc123\\")"},"first_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"First name of the contact"},"last_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Last name of the contact"},"email":{"type":"string","required":false,"visibility":"user-or-llm","description":"Email address"},"title":{"type":"string","required":false,"visibility":"user-or-llm","description":"Job title (e.g., \\"VP of Sales\\", \\"Software Engineer\\")"},"account_id":{"type":"string","required":false,"visibility":"user-or-llm","description":"Apollo account ID (e.g., \\"acc_abc123\\")"},"owner_id":{"type":"string","required":false,"visibility":"user-only","description":"User ID of the contact owner (accepted by Apollo but not officially documented for PATCH /contacts/{id})"},"organization_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Name of the contact\'s employer (e.g., \\"Apollo\\")"},"website_url":{"type":"string","required":false,"visibility":"user-or-llm","description":"Corporate website URL (e.g., \\"https://www.apollo.io/\\")"},"label_names":{"type":"array","required":false,"visibility":"user-or-llm","description":"Lists/labels to add the contact to (e.g., [\\"Prospects\\"])"},"contact_stage_id":{"type":"string","required":false,"visibility":"user-or-llm","description":"Apollo ID for the contact stage"},"present_raw_address":{"type":"string","required":false,"visibility":"user-or-llm","description":"Personal location for the contact (e.g., \\"Atlanta, United States\\")"},"direct_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Primary phone number"},"corporate_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Work/office phone number"},"mobile_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Mobile phone number"},"home_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Home phone number"},"other_phone":{"type":"string","required":false,"visibility":"user-or-llm","description":"Alternative phone number"},"typed_custom_fields":{"type":"json","required":false,"visibility":"user-or-llm","description":"Custom field values keyed by custom field ID"}},"hostedApiKey":"none"},"apollo_email_accounts":{"id":"apollo_email_accounts","name":"Apollo Get Email Accounts","description":"Get list of team\'s linked email accounts in Apollo","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"}},"hostedApiKey":"none"},"apollo_opportunity_create":{"id":"apollo_opportunity_create","name":"Apollo Create Opportunity","description":"Create a new deal for an account in your Apollo database (master key required)","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the opportunity/deal (e.g., \\"Enterprise License - Q1\\")"},"account_id":{"type":"string","required":false,"visibility":"user-or-llm","description":"ID of the account this opportunity belongs to (e.g., \\"acc_abc123\\")"},"amount":{"type":"string","required":false,"visibility":"user-or-llm","description":"Monetary value as a plain number string with no commas or currency symbols"},"opportunity_stage_id":{"type":"string","required":false,"visibility":"user-only","description":"ID of the opportunity stage"},"owner_id":{"type":"string","required":false,"visibility":"user-only","description":"User ID of the opportunity owner"},"closed_date":{"type":"string","required":false,"visibility":"user-or-llm","description":"Expected close date in YYYY-MM-DD format"},"typed_custom_fields":{"type":"json","required":false,"visibility":"user-only","description":"Custom field values as { custom_field_id: value } map"}},"hostedApiKey":"none"},"apollo_opportunity_get":{"id":"apollo_opportunity_get","name":"Apollo Get Opportunity","description":"Retrieve complete details of a specific deal/opportunity by ID","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"opportunity_id":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the opportunity to retrieve (e.g., \\"opp_abc123\\")"}},"hostedApiKey":"none"},"apollo_opportunity_search":{"id":"apollo_opportunity_search","name":"Apollo Search Opportunities","description":"Search and list all deals/opportunities in your team\'s Apollo account","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"sort_by_field":{"type":"string","required":false,"visibility":"user-or-llm","description":"Sort field: \\"amount\\", \\"is_closed\\", or \\"is_won\\""},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number for pagination (e.g., 1, 2, 3)"},"per_page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Results per page, max 100 (e.g., 25, 50, 100)"}},"hostedApiKey":"none"},"apollo_opportunity_update":{"id":"apollo_opportunity_update","name":"Apollo Update Opportunity","description":"Update an existing deal/opportunity in your Apollo database","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"opportunity_id":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the opportunity to update (e.g., \\"opp_abc123\\")"},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Name of the opportunity/deal (e.g., \\"Enterprise License - Q1\\")"},"amount":{"type":"string","required":false,"visibility":"user-or-llm","description":"Monetary value as a plain number string with no commas or currency symbols"},"opportunity_stage_id":{"type":"string","required":false,"visibility":"user-only","description":"ID of the opportunity stage"},"owner_id":{"type":"string","required":false,"visibility":"user-only","description":"User ID of the opportunity owner"},"closed_date":{"type":"string","required":false,"visibility":"user-or-llm","description":"Expected close date in YYYY-MM-DD format"},"typed_custom_fields":{"type":"json","required":false,"visibility":"user-only","description":"Custom field values as { custom_field_id: value } map"}},"hostedApiKey":"none"},"apollo_organization_bulk_enrich":{"id":"apollo_organization_bulk_enrich","name":"Apollo Bulk Organization Enrichment","description":"Enrich data for up to 10 organizations at once using Apollo","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"domains":{"type":"array","required":true,"visibility":"user-or-llm","description":"Array of company domains to enrich (max 10, no www. or @, e.g., [\\"apollo.io\\", \\"stripe.com\\"])"}},"hostedApiKey":"none"},"apollo_organization_enrich":{"id":"apollo_organization_enrich","name":"Apollo Organization Enrichment","description":"Enrich data for a single organization using Apollo","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"domain":{"type":"string","required":true,"visibility":"user-or-llm","description":"Company domain (e.g., \\"apollo.io\\", \\"acme.com\\")"}},"hostedApiKey":"none"},"apollo_organization_search":{"id":"apollo_organization_search","name":"Apollo Organization Search","description":"Search Apollo\'s database for companies using filters","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"organization_locations":{"type":"array","required":false,"visibility":"user-or-llm","description":"Company HQ locations (cities, US states, or countries)"},"organization_not_locations":{"type":"array","required":false,"visibility":"user-or-llm","description":"Exclude companies whose HQ is in these locations"},"organization_num_employees_ranges":{"type":"array","required":false,"visibility":"user-or-llm","description":"Employee count ranges as \\"min,max\\" strings (e.g., [\\"1,10\\", \\"250,500\\", \\"10000,20000\\"])"},"q_organization_keyword_tags":{"type":"array","required":false,"visibility":"user-or-llm","description":"Industry or keyword tags"},"q_organization_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Organization name to search for (e.g., \\"Acme\\", \\"TechCorp\\")"},"organization_ids":{"type":"array","required":false,"visibility":"user-or-llm","description":"Apollo organization IDs to include (e.g., [\\"5e66b6381e05b4008c8331b8\\"])"},"q_organization_domains_list":{"type":"array","required":false,"visibility":"user-or-llm","description":"Domain names to filter by (no www. or @, up to 1,000)"},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number for pagination (e.g., 1, 2, 3)"},"per_page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Results per page, max 100 (e.g., 25, 50, 100)"}},"hostedApiKey":"none"},"apollo_people_bulk_enrich":{"id":"apollo_people_bulk_enrich","name":"Apollo Bulk People Enrichment","description":"Enrich data for up to 10 people at once using Apollo","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"people":{"type":"array","required":true,"visibility":"user-or-llm","description":"Array of people to enrich (max 10)"},"reveal_personal_emails":{"type":"boolean","required":false,"visibility":"user-only","description":"Reveal personal email addresses (uses credits)"},"reveal_phone_number":{"type":"boolean","required":false,"visibility":"user-only","description":"Reveal phone numbers (uses credits, requires webhook_url)"},"webhook_url":{"type":"string","required":false,"visibility":"user-only","description":"Webhook URL for async phone number delivery (required when reveal_phone_number is true)"}},"hostedApiKey":"none"},"apollo_people_enrich":{"id":"apollo_people_enrich","name":"Apollo People Enrichment","description":"Enrich data for a single person using Apollo","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"first_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"First name of the person"},"last_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Last name of the person"},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Full name of the person (alternative to first_name/last_name)"},"id":{"type":"string","required":false,"visibility":"user-or-llm","description":"Apollo ID for the person"},"hashed_email":{"type":"string","required":false,"visibility":"user-or-llm","description":"MD5 or SHA-256 hashed email"},"email":{"type":"string","required":false,"visibility":"user-or-llm","description":"Email address of the person"},"organization_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Company name where the person works"},"domain":{"type":"string","required":false,"visibility":"user-or-llm","description":"Company domain (e.g., \\"apollo.io\\", \\"acme.com\\")"},"linkedin_url":{"type":"string","required":false,"visibility":"user-or-llm","description":"LinkedIn profile URL"},"reveal_personal_emails":{"type":"boolean","required":false,"visibility":"user-only","description":"Reveal personal email addresses (uses credits)"},"reveal_phone_number":{"type":"boolean","required":false,"visibility":"user-only","description":"Reveal phone numbers (uses credits, requires webhook_url)"},"webhook_url":{"type":"string","required":false,"visibility":"user-only","description":"Webhook URL for async phone number delivery (required when reveal_phone_number is true)"}},"hostedApiKey":"none"},"apollo_people_search":{"id":"apollo_people_search","name":"Apollo People Search","description":"Search Apollo\'s database for people using demographic filters","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key"},"person_titles":{"type":"array","required":false,"visibility":"user-or-llm","description":"Job titles to search for (e.g., [\\"CEO\\", \\"VP of Sales\\"])"},"include_similar_titles":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Whether to return people with job titles similar to person_titles"},"person_locations":{"type":"array","required":false,"visibility":"user-or-llm","description":"Locations to search in (e.g., [\\"San Francisco, CA\\", \\"New York, NY\\"])"},"person_seniorities":{"type":"array","required":false,"visibility":"user-or-llm","description":"Seniority levels (one of: owner, founder, c_suite, partner, vp, head, director, manager, senior, entry, intern)"},"organization_ids":{"type":"array","required":false,"visibility":"user-or-llm","description":"Apollo organization IDs to filter by (e.g., [\\"5e66b6381e05b4008c8331b8\\"])"},"organization_names":{"type":"array","required":false,"visibility":"user-or-llm","description":"Company names to search within (legacy filter)"},"organization_locations":{"type":"array","required":false,"visibility":"user-or-llm","description":"Headquarters locations of the people\'s current employer (e.g., [\'texas\', \'tokyo\', \'spain\'])"},"q_organization_domains_list":{"type":"array","required":false,"visibility":"user-or-llm","description":"Employer domain names (e.g., [\\"apollo.io\\", \\"microsoft.com\\"]) — up to 1,000, no www. or @"},"organization_num_employees_ranges":{"type":"array","required":false,"visibility":"user-or-llm","description":"Employee count ranges for the person\'s current employer. Each entry is \\"min,max\\" (e.g., [\\"1,10\\", \\"250,500\\", \\"10000,20000\\"])"},"contact_email_status":{"type":"array","required":false,"visibility":"user-or-llm","description":"Email statuses to filter by: \\"verified\\", \\"unverified\\", \\"likely to engage\\", \\"unavailable\\""},"q_keywords":{"type":"string","required":false,"visibility":"user-or-llm","description":"Keywords to search for"},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number for pagination, default 1 (e.g., 1, 2, 3)"},"per_page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Results per page, default 25, max 100 (e.g., 25, 50, 100)"}},"hostedApiKey":"none"},"apollo_sequence_add_contacts":{"id":"apollo_sequence_add_contacts","name":"Apollo Add Contacts to Sequence","description":"Add contacts to an Apollo sequence","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"sequence_id":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the sequence to add contacts to (e.g., \\"seq_abc123\\")"},"contact_ids":{"type":"array","required":false,"visibility":"user-or-llm","description":"Array of contact IDs to add to the sequence (e.g., [\\"con_abc123\\", \\"con_def456\\"]). Either contact_ids or label_names must be provided."},"label_names":{"type":"array","required":false,"visibility":"user-or-llm","description":"Array of label names to identify contacts to add to the sequence. Either contact_ids or label_names must be provided."},"send_email_from_email_account_id":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the email account to send from. Use the Get Email Accounts operation to look this up."},"send_email_from_email_address":{"type":"string","required":false,"visibility":"user-only","description":"Specific email address to send from within the email account."},"sequence_no_email":{"type":"boolean","required":false,"visibility":"user-only","description":"Add contacts even if they have no email address"},"sequence_unverified_email":{"type":"boolean","required":false,"visibility":"user-only","description":"Add contacts with unverified email addresses"},"sequence_job_change":{"type":"boolean","required":false,"visibility":"user-only","description":"Add contacts who recently changed jobs"},"sequence_active_in_other_campaigns":{"type":"boolean","required":false,"visibility":"user-only","description":"Add contacts active in other campaigns"},"sequence_finished_in_other_campaigns":{"type":"boolean","required":false,"visibility":"user-only","description":"Add contacts who finished other campaigns"},"sequence_same_company_in_same_campaign":{"type":"boolean","required":false,"visibility":"user-only","description":"Add contacts even if others from the same company are in the sequence"},"contacts_without_ownership_permission":{"type":"boolean","required":false,"visibility":"user-only","description":"Add contacts without ownership permission"},"add_if_in_queue":{"type":"boolean","required":false,"visibility":"user-only","description":"Add contacts even if they are in the queue"},"contact_verification_skipped":{"type":"boolean","required":false,"visibility":"user-only","description":"Skip contact verification when adding"},"user_id":{"type":"string","required":false,"visibility":"user-only","description":"ID of the user performing the action"},"status":{"type":"string","required":false,"visibility":"user-only","description":"Initial status for added contacts: \\"active\\" or \\"paused\\""},"auto_unpause_at":{"type":"string","required":false,"visibility":"user-only","description":"ISO 8601 datetime to automatically unpause contacts"}},"hostedApiKey":"none"},"apollo_sequence_search":{"id":"apollo_sequence_search","name":"Apollo Search Sequences","description":"Search for sequences/campaigns in your team\'s Apollo account (master key required)","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"q_name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Search sequences by name (e.g., \\"Outbound Q1\\", \\"Follow-up\\")"},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number for pagination (e.g., 1, 2, 3)"},"per_page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Results per page, max 100 (e.g., 25, 50, 100)"}},"hostedApiKey":"none"},"apollo_task_create":{"id":"apollo_task_create","name":"Apollo Create Task","description":"Create one or more tasks in Apollo (one task per contact_id, master key required)","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"user_id":{"type":"string","required":true,"visibility":"user-or-llm","description":"ID of the Apollo user the task is assigned to"},"contact_ids":{"type":"array","required":true,"visibility":"user-or-llm","description":"Array of contact IDs. One task is created per contact."},"priority":{"type":"string","required":false,"visibility":"user-or-llm","description":"Task priority: \\"high\\", \\"medium\\", or \\"low\\" (defaults to \\"medium\\")"},"due_at":{"type":"string","required":true,"visibility":"user-or-llm","description":"Due date/time in ISO 8601 format (e.g., \\"2024-12-31T23:59:59Z\\")"},"type":{"type":"string","required":true,"visibility":"user-or-llm","description":"Task type: \\"call\\", \\"outreach_manual_email\\", \\"linkedin_step_connect\\", \\"linkedin_step_message\\", \\"linkedin_step_view_profile\\", \\"linkedin_step_interact_post\\", or \\"action_item\\""},"status":{"type":"string","required":true,"visibility":"user-or-llm","description":"Task status: \\"scheduled\\", \\"completed\\", or \\"skipped\\""},"note":{"type":"string","required":false,"visibility":"user-or-llm","description":"Free-form note providing context for the task"}},"hostedApiKey":"none"},"apollo_task_search":{"id":"apollo_task_search","name":"Apollo Search Tasks","description":"Search for tasks in Apollo","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Apollo API key (master key required)"},"sort_by_field":{"type":"string","required":false,"visibility":"user-or-llm","description":"Sort field: \\"task_due_at\\" or \\"task_priority\\""},"open_factor_names":{"type":"array","required":false,"visibility":"user-or-llm","description":"Filter by status. Common values: [\\"task_types\\"] for open tasks, [\\"task_completed_at\\"] for completed tasks."},"page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Page number for pagination (e.g., 1, 2, 3)"},"per_page":{"type":"number","required":false,"visibility":"user-or-llm","description":"Results per page, max 100 (e.g., 25, 50, 100)"}},"hostedApiKey":"none"},"appconfig_create_application":{"id":"appconfig_create_application","name":"AppConfig Create Application","description":"Create an application in AWS AppConfig","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the application to create"},"description":{"type":"string","required":false,"visibility":"user-or-llm","description":"Description of the application"}},"hostedApiKey":"none"},"appconfig_create_configuration_profile":{"id":"appconfig_create_configuration_profile","name":"AppConfig Create Configuration Profile","description":"Create a configuration profile in an AWS AppConfig application","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID to create the configuration profile in"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the configuration profile"},"locationUri":{"type":"string","required":true,"visibility":"user-or-llm","description":"Where the configuration is stored. Use \\"hosted\\" for AppConfig-hosted configurations, or an SSM/S3 URI"},"description":{"type":"string","required":false,"visibility":"user-or-llm","description":"Description of the configuration profile"},"retrievalRoleArn":{"type":"string","required":false,"visibility":"user-or-llm","description":"ARN of an IAM role to retrieve the configuration (required for non-hosted URIs)"},"type":{"type":"string","required":false,"visibility":"user-or-llm","description":"Profile type: AWS.Freeform (default) or AWS.AppConfig.FeatureFlags"}},"hostedApiKey":"none"},"appconfig_create_environment":{"id":"appconfig_create_environment","name":"AppConfig Create Environment","description":"Create an environment for an AWS AppConfig application","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID to create the environment in"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the environment to create"},"description":{"type":"string","required":false,"visibility":"user-or-llm","description":"Description of the environment"}},"hostedApiKey":"none"},"appconfig_create_hosted_configuration_version":{"id":"appconfig_create_hosted_configuration_version","name":"AppConfig Create Hosted Configuration Version","description":"Create a new hosted configuration version for an AppConfig configuration profile","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the configuration profile"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID to add the version to"},"content":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration content (e.g., a JSON or YAML document)"},"contentType":{"type":"string","required":true,"visibility":"user-or-llm","description":"Content type of the configuration (e.g., application/json, text/plain)"},"description":{"type":"string","required":false,"visibility":"user-or-llm","description":"Description of the configuration version"},"latestVersionNumber":{"type":"number","required":false,"visibility":"user-or-llm","description":"The version number of the latest version, used for optimistic concurrency"},"versionLabel":{"type":"string","required":false,"visibility":"user-or-llm","description":"A user-defined label for the configuration version"}},"hostedApiKey":"none"},"appconfig_delete_application":{"id":"appconfig_delete_application","name":"AppConfig Delete Application","description":"Delete an AWS AppConfig application","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID to delete"}},"hostedApiKey":"none"},"appconfig_delete_configuration_profile":{"id":"appconfig_delete_configuration_profile","name":"AppConfig Delete Configuration Profile","description":"Delete an AWS AppConfig configuration profile","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the configuration profile"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID to delete"}},"hostedApiKey":"none"},"appconfig_delete_environment":{"id":"appconfig_delete_environment","name":"AppConfig Delete Environment","description":"Delete an AWS AppConfig environment","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the environment"},"environmentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The environment ID to delete"}},"hostedApiKey":"none"},"appconfig_delete_hosted_configuration_version":{"id":"appconfig_delete_hosted_configuration_version","name":"AppConfig Delete Hosted Configuration Version","description":"Delete a specific hosted configuration version from an AppConfig profile","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the configuration profile"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID that owns the version"},"versionNumber":{"type":"number","required":true,"visibility":"user-or-llm","description":"The version number to delete"}},"hostedApiKey":"none"},"appconfig_get_application":{"id":"appconfig_get_application","name":"AppConfig Get Application","description":"Get details about a single AWS AppConfig application","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID to retrieve"}},"hostedApiKey":"none"},"appconfig_get_configuration":{"id":"appconfig_get_configuration","name":"AppConfig Get Configuration","description":"Retrieve the latest deployed configuration for an AppConfig application, environment, and profile","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID or name to retrieve configuration for"},"environmentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The environment ID or name to retrieve configuration for"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID or name to retrieve"}},"hostedApiKey":"none"},"appconfig_get_configuration_profile":{"id":"appconfig_get_configuration_profile","name":"AppConfig Get Configuration Profile","description":"Get details about a single AWS AppConfig configuration profile","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the configuration profile"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID to retrieve"}},"hostedApiKey":"none"},"appconfig_get_deployment":{"id":"appconfig_get_deployment","name":"AppConfig Get Deployment","description":"Get details about a specific AWS AppConfig deployment","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID of the deployment"},"environmentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The environment ID of the deployment"},"deploymentNumber":{"type":"number","required":true,"visibility":"user-or-llm","description":"The sequence number of the deployment"}},"hostedApiKey":"none"},"appconfig_get_environment":{"id":"appconfig_get_environment","name":"AppConfig Get Environment","description":"Get details about a single AWS AppConfig environment","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the environment"},"environmentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The environment ID to retrieve"}},"hostedApiKey":"none"},"appconfig_get_hosted_configuration_version":{"id":"appconfig_get_hosted_configuration_version","name":"AppConfig Get Hosted Configuration Version","description":"Retrieve a specific hosted configuration version from an AppConfig profile","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the configuration profile"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID to read the version from"},"versionNumber":{"type":"number","required":true,"visibility":"user-or-llm","description":"The version number to retrieve"}},"hostedApiKey":"none"},"appconfig_list_applications":{"id":"appconfig_list_applications","name":"AppConfig List Applications","description":"List applications in AWS AppConfig","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"maxResults":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of applications to return (1-50)"},"nextToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token from a previous response"}},"hostedApiKey":"none"},"appconfig_list_configuration_profiles":{"id":"appconfig_list_configuration_profiles","name":"AppConfig List Configuration Profiles","description":"List configuration profiles for an AWS AppConfig application","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the configuration profiles"},"maxResults":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of configuration profiles to return (1-50)"},"nextToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token from a previous response"}},"hostedApiKey":"none"},"appconfig_list_deployment_strategies":{"id":"appconfig_list_deployment_strategies","name":"AppConfig List Deployment Strategies","description":"List deployment strategies available in AWS AppConfig","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"maxResults":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of deployment strategies to return (1-50)"},"nextToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token from a previous response"}},"hostedApiKey":"none"},"appconfig_list_deployments":{"id":"appconfig_list_deployments","name":"AppConfig List Deployments","description":"List deployments for an AWS AppConfig environment","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID of the deployments"},"environmentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The environment ID of the deployments"},"maxResults":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of deployments to return (1-50)"},"nextToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token from a previous response"}},"hostedApiKey":"none"},"appconfig_list_environments":{"id":"appconfig_list_environments","name":"AppConfig List Environments","description":"List environments for an AWS AppConfig application","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the environments"},"maxResults":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of environments to return (1-50)"},"nextToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token from a previous response"}},"hostedApiKey":"none"},"appconfig_list_hosted_configuration_versions":{"id":"appconfig_list_hosted_configuration_versions","name":"AppConfig List Hosted Configuration Versions","description":"List hosted configuration versions for an AWS AppConfig configuration profile","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the configuration profile"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID to list versions for"},"maxResults":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of versions to return (1-50)"},"nextToken":{"type":"string","required":false,"visibility":"user-or-llm","description":"Pagination token from a previous response"}},"hostedApiKey":"none"},"appconfig_start_deployment":{"id":"appconfig_start_deployment","name":"AppConfig Start Deployment","description":"Start deploying a configuration version to an AWS AppConfig environment","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID to deploy in"},"environmentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The environment ID to deploy to"},"deploymentStrategyId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The deployment strategy ID to use"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID to deploy"},"configurationVersion":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration version to deploy"},"description":{"type":"string","required":false,"visibility":"user-or-llm","description":"Description of the deployment"}},"hostedApiKey":"none"},"appconfig_stop_deployment":{"id":"appconfig_stop_deployment","name":"AppConfig Stop Deployment","description":"Stop an in-progress AWS AppConfig deployment","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID of the deployment"},"environmentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The environment ID of the deployment"},"deploymentNumber":{"type":"number","required":true,"visibility":"user-or-llm","description":"The sequence number of the deployment to stop"}},"hostedApiKey":"none"},"appconfig_update_application":{"id":"appconfig_update_application","name":"AppConfig Update Application","description":"Update the name or description of an AWS AppConfig application","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID to update"},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"New name for the application"},"description":{"type":"string","required":false,"visibility":"user-or-llm","description":"New description for the application"}},"hostedApiKey":"none"},"appconfig_update_configuration_profile":{"id":"appconfig_update_configuration_profile","name":"AppConfig Update Configuration Profile","description":"Update the name, description, or retrieval role of an AppConfig configuration profile","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the configuration profile"},"configurationProfileId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The configuration profile ID to update"},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"New name for the configuration profile"},"description":{"type":"string","required":false,"visibility":"user-or-llm","description":"New description for the configuration profile"},"retrievalRoleArn":{"type":"string","required":false,"visibility":"user-or-llm","description":"New ARN of the IAM role used to retrieve the configuration"}},"hostedApiKey":"none"},"appconfig_update_environment":{"id":"appconfig_update_environment","name":"AppConfig Update Environment","description":"Update the name or description of an AWS AppConfig environment","version":"1.0","params":{"region":{"type":"string","required":true,"visibility":"user-only","description":"AWS region (e.g., us-east-1)"},"accessKeyId":{"type":"string","required":true,"visibility":"user-only","description":"AWS access key ID"},"secretAccessKey":{"type":"string","required":true,"visibility":"user-only","description":"AWS secret access key"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The application ID that owns the environment"},"environmentId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The environment ID to update"},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"New name for the environment"},"description":{"type":"string","required":false,"visibility":"user-or-llm","description":"New description for the environment"}},"hostedApiKey":"none"},"arxiv_get_author_papers":{"id":"arxiv_get_author_papers","name":"ArXiv Get Author Papers","description":"Search for papers by a specific author on ArXiv.","version":"1.0.0","params":{"authorName":{"type":"string","required":true,"visibility":"user-or-llm","description":"Author name to search for"},"maxResults":{"type":"number","required":false,"visibility":"user-only","description":"Maximum number of results to return (default: 10, max: 2000)"}},"hostedApiKey":"none"},"arxiv_get_paper":{"id":"arxiv_get_paper","name":"ArXiv Get Paper","description":"Get detailed information about a specific ArXiv paper by its ID.","version":"1.0.0","params":{"paperId":{"type":"string","required":true,"visibility":"user-or-llm","description":"ArXiv paper ID (e.g., \\"1706.03762\\")"}},"hostedApiKey":"none"},"arxiv_search":{"id":"arxiv_search","name":"ArXiv Search","description":"Search for academic papers on ArXiv by keywords, authors, titles, or other fields.","version":"1.0.0","params":{"searchQuery":{"type":"string","required":true,"visibility":"user-or-llm","description":"The search query to execute"},"searchField":{"type":"string","required":false,"visibility":"user-only","description":"Field to search in: all, ti (title), au (author), abs (abstract), co (comment), jr (journal), cat (category), rn (report number)"},"maxResults":{"type":"number","required":false,"visibility":"user-only","description":"Maximum number of results to return (default: 10, max: 2000)"},"sortBy":{"type":"string","required":false,"visibility":"user-only","description":"Sort by: relevance, lastUpdatedDate, submittedDate (default: relevance)"},"sortOrder":{"type":"string","required":false,"visibility":"user-only","description":"Sort order: ascending, descending (default: descending)"}},"hostedApiKey":"none"},"asana_add_comment":{"id":"asana_add_comment","name":"Asana Add Comment","description":"Add a comment (story) to an Asana task","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"taskGid":{"type":"string","required":true,"visibility":"user-or-llm","description":"Asana task GID (numeric string)"},"text":{"type":"string","required":true,"visibility":"user-or-llm","description":"The text content of the comment"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"asana_add_followers":{"id":"asana_add_followers","name":"Asana Add Followers","description":"Add one or more followers to an Asana task","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"taskGid":{"type":"string","required":true,"visibility":"user-or-llm","description":"GID of the Asana task (numeric string)"},"followers":{"type":"array","required":true,"visibility":"user-or-llm","description":"Array of user GIDs to add as followers to the task"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"asana_create_project":{"id":"asana_create_project","name":"Asana Create Project","description":"Create a new project in an Asana workspace","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"workspace":{"type":"string","required":true,"visibility":"user-or-llm","description":"Asana workspace GID (numeric string) where the project will be created"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the project"},"notes":{"type":"string","required":false,"visibility":"user-or-llm","description":"Notes or description for the project"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"asana_create_section":{"id":"asana_create_section","name":"Asana Create Section","description":"Create a new section in an Asana project","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"projectGid":{"type":"string","required":true,"visibility":"user-or-llm","description":"GID of the Asana project (numeric string) to add the section to"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the section"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"asana_create_subtask":{"id":"asana_create_subtask","name":"Asana Create Subtask","description":"Create a subtask under an existing Asana task","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"taskGid":{"type":"string","required":true,"visibility":"user-or-llm","description":"GID of the parent Asana task (numeric string)"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the subtask"},"notes":{"type":"string","required":false,"visibility":"user-or-llm","description":"Notes or description for the subtask"},"assignee":{"type":"string","required":false,"visibility":"user-or-llm","description":"User GID to assign the subtask to"},"due_on":{"type":"string","required":false,"visibility":"user-or-llm","description":"Due date in YYYY-MM-DD format"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"asana_create_task":{"id":"asana_create_task","name":"Asana Create Task","description":"Create a new task in Asana","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"workspace":{"type":"string","required":true,"visibility":"user-or-llm","description":"Asana workspace GID (numeric string) where the task will be created"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"Name of the task"},"notes":{"type":"string","required":false,"visibility":"user-or-llm","description":"Notes or description for the task"},"assignee":{"type":"string","required":false,"visibility":"user-or-llm","description":"User GID to assign the task to"},"due_on":{"type":"string","required":false,"visibility":"user-or-llm","description":"Due date in YYYY-MM-DD format"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"asana_delete_task":{"id":"asana_delete_task","name":"Asana Delete Task","description":"Delete an Asana task by its GID (moves it to the trash)","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"taskGid":{"type":"string","required":true,"visibility":"user-or-llm","description":"GID of the Asana task to delete (numeric string)"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"asana_get_project":{"id":"asana_get_project","name":"Asana Get Project","description":"Retrieve a single Asana project by its GID","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"projectGid":{"type":"string","required":true,"visibility":"user-or-llm","description":"Asana project GID (numeric string) to retrieve"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"asana_get_projects":{"id":"asana_get_projects","name":"Asana Get Projects","description":"Retrieve all projects from an Asana workspace","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"workspace":{"type":"string","required":true,"visibility":"user-or-llm","description":"Asana workspace GID (numeric string) to retrieve projects from"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"asana_get_task":{"id":"asana_get_task","name":"Asana Get Task","description":"Retrieve a single task by GID or get multiple tasks with filters","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"taskGid":{"type":"string","required":false,"visibility":"user-or-llm","description":"The globally unique identifier (GID) of the task. If not provided, will get multiple tasks."},"workspace":{"type":"string","required":false,"visibility":"user-or-llm","description":"Asana workspace GID (numeric string) to filter tasks (required when not using taskGid)"},"project":{"type":"string","required":false,"visibility":"user-or-llm","description":"Asana project GID (numeric string) to filter tasks"},"limit":{"type":"number","required":false,"visibility":"user-or-llm","description":"Maximum number of tasks to return (default: 50)"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"asana_list_sections":{"id":"asana_list_sections","name":"Asana List Sections","description":"List all sections in an Asana project","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"projectGid":{"type":"string","required":true,"visibility":"user-or-llm","description":"GID of the Asana project (numeric string) to list sections from"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"asana_list_workspaces":{"id":"asana_list_workspaces","name":"Asana List Workspaces","description":"List all Asana workspaces and organizations the authenticated user belongs to","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"asana_search_tasks":{"id":"asana_search_tasks","name":"Asana Search Tasks","description":"Search for tasks in an Asana workspace","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"workspace":{"type":"string","required":true,"visibility":"user-or-llm","description":"Asana workspace GID (numeric string) to search tasks in"},"text":{"type":"string","required":false,"visibility":"user-or-llm","description":"Text to search for in task names"},"assignee":{"type":"string","required":false,"visibility":"user-or-llm","description":"Filter tasks by assignee user GID"},"projects":{"type":"array","required":false,"visibility":"user-or-llm","description":"Array of Asana project GIDs (numeric strings) to filter tasks by"},"completed":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Filter by completion status"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"asana_update_task":{"id":"asana_update_task","name":"Asana Update Task","description":"Update an existing task in Asana","version":"1.0.0","params":{"accessToken":{"type":"string","required":true,"visibility":"hidden","description":"OAuth access token for Asana"},"taskGid":{"type":"string","required":true,"visibility":"user-or-llm","description":"Asana task GID (numeric string) of the task to update"},"name":{"type":"string","required":false,"visibility":"user-or-llm","description":"Updated name for the task"},"notes":{"type":"string","required":false,"visibility":"user-or-llm","description":"Updated notes or description for the task"},"assignee":{"type":"string","required":false,"visibility":"user-or-llm","description":"Updated assignee user GID"},"completed":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Mark task as completed or not completed"},"due_on":{"type":"string","required":false,"visibility":"user-or-llm","description":"Updated due date in YYYY-MM-DD format"}},"oauth":{"required":true,"provider":"asana"},"hostedApiKey":"none"},"ashby_add_candidate_tag":{"id":"ashby_add_candidate_tag","name":"Ashby Add Candidate Tag","description":"Adds a tag to a candidate in Ashby and returns the updated candidate.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ashby API Key"},"onBehalfOfUserId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls"},"candidateId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The UUID of the candidate to add the tag to"},"tagId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The UUID of the tag to add"}},"hostedApiKey":"none"},"ashby_anonymize_candidate":{"id":"ashby_anonymize_candidate","name":"Ashby Anonymize Candidate","description":"Strips personally identifiable information from a candidate in Ashby. This does not delete the candidate - the record and its applications remain, with the PII removed. Ashby exposes no candidate deletion endpoint; true deletion is UI-only, restricted by role, and limited to a 10-day window. Requires the candidatesWrite permission.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ashby API Key"},"onBehalfOfUserId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls"},"candidateId":{"type":"string","required":true,"visibility":"user-or-llm","description":"UUID of the candidate to anonymize"}},"hostedApiKey":"none"},"ashby_change_application_source":{"id":"ashby_change_application_source","name":"Ashby Change Application Source","description":"Changes the source attributed to an existing application, so programmatically created applications report correctly on the recruiting side. Requires the candidatesWrite permission.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ashby API Key"},"onBehalfOfUserId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"UUID of the application whose source should change"},"sourceId":{"type":"string","required":false,"visibility":"user-or-llm","description":"UUID of the source to attribute the application to, as returned by List Sources. Omit only when unsetSource is true."},"unsetSource":{"type":"boolean","required":false,"visibility":"user-or-llm","description":"Set true to deliberately clear the application source. Required to unset, so that a missing or empty sourceId cannot wipe attribution by accident."}},"hostedApiKey":"none"},"ashby_change_application_stage":{"id":"ashby_change_application_stage","name":"Ashby Change Application Stage","description":"Moves an application to a different interview stage. Requires an archive reason when moving to an Archived stage.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ashby API Key"},"onBehalfOfUserId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls"},"applicationId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The UUID of the application to update the stage of"},"interviewStageId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The UUID of the interview stage to move the application to"},"archiveReasonId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Archive reason UUID. Required when moving to an Archived stage, ignored otherwise"},"archiveEmail":{"type":"json","required":false,"visibility":"user-or-llm","description":"Archive email configuration with communicationTemplateId and optional sendAt ISO 8601 timestamp. Pass null or omit to send no archive email."}},"hostedApiKey":"none"},"ashby_create_application":{"id":"ashby_create_application","name":"Ashby Create Application","description":"Creates a new application for a candidate on a job. Optionally specify interview plan, stage, source, and credited user.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ashby API Key"},"onBehalfOfUserId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls"},"candidateId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The UUID of the candidate to consider for the job"},"jobId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The UUID of the job to consider the candidate for"},"interviewPlanId":{"type":"string","required":false,"visibility":"user-or-llm","description":"UUID of the interview plan to use (defaults to the job default plan)"},"interviewStageId":{"type":"string","required":false,"visibility":"user-or-llm","description":"UUID of the interview stage to place the application in, or FirstPreInterviewScreen (defaults to the first Lead stage)"},"sourceId":{"type":"string","required":false,"visibility":"user-or-llm","description":"UUID of the source to set on the application"},"creditedToUserId":{"type":"string","required":false,"visibility":"user-or-llm","description":"UUID of the user the application is credited to"},"createdAt":{"type":"string","required":false,"visibility":"user-or-llm","description":"ISO 8601 timestamp to set as the application creation date (defaults to now)"},"applicationHistory":{"type":"json","required":false,"visibility":"user-or-llm","description":"Optional documented application history entries to create with the application"}},"hostedApiKey":"none"},"ashby_create_candidate":{"id":"ashby_create_candidate","name":"Ashby Create Candidate","description":"Creates a new candidate record in Ashby.","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ashby API Key"},"onBehalfOfUserId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls"},"name":{"type":"string","required":true,"visibility":"user-or-llm","description":"The candidate full name"},"email":{"type":"string","required":false,"visibility":"user-or-llm","description":"Primary email address for the candidate"},"phoneNumber":{"type":"string","required":false,"visibility":"user-or-llm","description":"Primary phone number for the candidate"},"linkedInUrl":{"type":"string","required":false,"visibility":"user-or-llm","description":"LinkedIn profile URL"},"githubUrl":{"type":"string","required":false,"visibility":"user-or-llm","description":"GitHub profile URL"},"website":{"type":"string","required":false,"visibility":"user-or-llm","description":"Personal website URL"},"sourceId":{"type":"string","required":false,"visibility":"user-or-llm","description":"UUID of the source to attribute the candidate to"},"creditedToUserId":{"type":"string","required":false,"visibility":"user-or-llm","description":"UUID of the Ashby user to credit with sourcing this candidate"},"createdAt":{"type":"string","required":false,"visibility":"user-or-llm","description":"Backdated creation timestamp in ISO 8601 (e.g. 2024-01-01T00:00:00Z). Defaults to now."},"alternateEmailAddresses":{"type":"json","required":false,"visibility":"user-or-llm","description":"Array of additional email address strings to add to the candidate, e.g. [\\"a@x.com\\",\\"b@y.com\\"]"},"location":{"type":"json","required":false,"visibility":"user-or-llm","description":"Candidate location object with optional city, region, and country"}},"hostedApiKey":"none"},"ashby_create_note":{"id":"ashby_create_note","name":"Ashby Create Note","description":"Creates a note on a candidate in Ashby. Supports plain text and HTML content (bold, italic, underline, links, lists, code).","version":"1.0.0","params":{"apiKey":{"type":"string","required":true,"visibility":"user-only","description":"Ashby API Key"},"onBehalfOfUserId":{"type":"string","required":false,"visibility":"user-or-llm","description":"Active Ashby user UUID to attribute this mutation to; the API key must permit on-behalf-of calls"},"candidateId":{"type":"string","required":true,"visibility":"user-or-llm","description":"The UUID of the candidate to add the note to"},"note":{"type":"string","required":true,"visibility":"user-or-llm","description":"The note content. If noteType is text/html, supports: , , , ,