feat(knowledge): crawl Slack per member on Sim Search through each person's own Slack user token

Author: waleedlatif1

apps/sim/connectors/permission-scoped-listing.test.ts

@@ -3,12 +3,29 @@
  */
 import { describe, expect, it } from 'vitest'
 import { getManagedOAuthConnectorPolicy } from '@/lib/auth/connectors/managed-oauth'
+import { getCredentialGroupProviderAdapter } from '@/lib/credential-groups/provider-registry'
 import {
+  type CredentialGroupProvider,
+  getCredentialGroupProviderFromProviderId,
   getCredentialGroupProviderService,
-  getCredentialGroupStandardOAuthProviderFromProviderId,
+  isCredentialGroupStandardOAuthProvider,
 } from '@/lib/credential-groups/providers'
+import { SLACK_MANAGED_USER_SCOPES } from '@/lib/credential-groups/slack-managed-user-scopes'
 import { CONNECTOR_META_REGISTRY } from '@/connectors/registry'
 
+/**
+ * The scopes an option of the provider requests from every member: the
+ * provider's service scopes plus its managed policy's additions for a standard
+ * OAuth provider, and the fixed user-token policy for Slack.
+ */
+function optionScopesFor(provider: CredentialGroupProvider): string[] {
+  if (!isCredentialGroupStandardOAuthProvider(provider)) return [...SLACK_MANAGED_USER_SCOPES]
+  const service = getCredentialGroupProviderService(provider)
+  const policy = getManagedOAuthConnectorPolicy(service.providerId)
+  expect(policy).toBeDefined()
+  return [...new Set([...service.scopes, ...(policy?.additionalScopes ?? [])])]
+}
+
 const permissionScoped = Object.values(CONNECTOR_META_REGISTRY).filter(
   (meta) => meta.permissionScopedListing !== undefined
 )
@@ -50,32 +67,22 @@ describe('permission-scoped connector listings', () => {
       'outlook',
       'salesforce',
       'sharepoint',
+      'slack',
       'zoom',
     ])
   })
 
   it.each(permissionScoped.map((meta) => [meta.id, meta] as const))(
-    '%s authenticates through a managed OAuth provider whose option scopes cover its read scopes',
+    '%s authenticates through a Credential Group provider whose option scopes cover its read scopes',
     (_id, meta) => {
       expect(meta.auth.mode).toBe('oauth')
       if (meta.auth.mode !== 'oauth') return
 
-      const policy = getManagedOAuthConnectorPolicy(meta.auth.provider)
-      expect(policy).toBeDefined()
-      if (!policy) return
-
-      const groupProvider = getCredentialGroupStandardOAuthProviderFromProviderId(
-        meta.auth.provider
-      )
-      expect(groupProvider).toBeDefined()
-
-      const optionScopes = [
-        ...new Set([
-          ...getCredentialGroupProviderService(groupProvider).scopes,
-          ...policy.additionalScopes,
-        ]),
-      ]
-      expect(policy.hasRequiredScopes(optionScopes, meta.auth.requiredScopes ?? [])).toBe(true)
+      const provider = getCredentialGroupProviderFromProviderId(meta.auth.provider)
+      const adapter = getCredentialGroupProviderAdapter(provider)
+      expect(
+        adapter.hasRequiredScopes(optionScopesFor(provider), meta.auth.requiredScopes ?? [])
+      ).toBe(true)
     }
   )
 

apps/sim/connectors/slack/meta.ts

@@ -22,6 +22,17 @@ export const slackConnectorMeta: ConnectorMeta = {
     ],
   },
 
+  /**
+   * `conversations.list` under a person's own token returns the public
+   * channels of their workspace and the private channels they belong to,
+   * exactly what they may read, so one member's crawl is their access. The
+   * channel selection is a cap: it would hide part of a member's corpus, and
+   * the per-member crawl indexes every channel the member can see instead.
+   * `maxMessages` bounds each channel document's window, not which channels
+   * are listed, so it is not a cap.
+   */
+  permissionScopedListing: { capFieldIds: ['channel'] },
+
   configFields: [
     {
       id: 'channelSelector',

apps/sim/connectors/slack/slack.test.ts

@@ -0,0 +1,191 @@
+/**
+ * @vitest-environment node
+ */
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { slackConnectorMeta } from '@/connectors/slack/meta'
+import { slackConnector } from '@/connectors/slack/slack'
+import { CONNECTOR_TEXT_DOCUMENT_MAX_BYTES, PER_MEMBER_LISTING_CONTEXT } from '@/connectors/utils'
+
+const GENERAL = {
+  id: 'C0GENERAL',
+  name: 'general',
+  topic: { value: 'Company-wide announcements' },
+  purpose: { value: '' },
+}
+const PLATFORM = { id: 'G0PLATFORM', name: 'platform', topic: { value: '' } }
+
+const MESSAGES = [
+  { type: 'message', user: 'U2', text: 'Shipping today', ts: '1700000200.000100' },
+  { type: 'message', user: 'U1', text: 'Morning', ts: '1700000100.000100' },
+  { type: 'message', subtype: 'channel_join', user: 'U1', text: 'joined', ts: '1700000000.000100' },
+]
+
+function jsonResponse(body: unknown): Response {
+  return new Response(JSON.stringify(body), {
+    status: 200,
+    headers: { 'Content-Type': 'application/json' },
+  })
+}
+
+const requestedUrls: string[] = []
+const fetchMock = vi.fn<(input: string | URL | Request, init?: RequestInit) => Promise<Response>>()
+
+/** Channels returned by `conversations.list`; per-test overridable. */
+let listedChannels: Record<string, unknown>[] = [GENERAL, PLATFORM]
+/** `next_cursor` returned by `conversations.list`; per-test overridable. */
+let listNextCursor = ''
+/** Messages returned by `conversations.history`; per-test overridable. */
+let history: Record<string, unknown>[] = MESSAGES
+/** Whether `conversations.info` reports the channel as missing; per-test overridable. */
+let channelMissing = false
+
+beforeEach(() => {
+  requestedUrls.length = 0
+  listedChannels = [GENERAL, PLATFORM]
+  listNextCursor = ''
+  history = MESSAGES
+  channelMissing = false
+  fetchMock.mockReset()
+  fetchMock.mockImplementation(async (input) => {
+    const url = new URL(String(input))
+    requestedUrls.push(`${url.pathname}?${url.searchParams.toString()}`)
+    switch (url.pathname) {
+      case '/api/auth.test':
+        return jsonResponse({ ok: true, team_id: 'T0TEAM' })
+      case '/api/conversations.list':
+        return jsonResponse({
+          ok: true,
+          channels: listedChannels,
+          response_metadata: { next_cursor: listNextCursor },
+        })
+      case '/api/conversations.info':
+        return channelMissing
+          ? jsonResponse({ ok: false, error: 'channel_not_found' })
+          : jsonResponse({ ok: true, channel: GENERAL })
+      case '/api/conversations.history':
+        return jsonResponse({ ok: true, messages: history, response_metadata: {} })
+      case '/api/users.info': {
+        const id = url.searchParams.get('user')
+        return jsonResponse({ ok: true, user: { id, name: id, real_name: `Person ${id}` } })
+      }
+      default:
+        return jsonResponse({ ok: false, error: 'unknown_method' })
+    }
+  })
+  vi.stubGlobal('fetch', fetchMock)
+})
+
+afterEach(() => {
+  vi.unstubAllGlobals()
+})
+
+const requested = (method: string) => requestedUrls.filter((url) => url.includes(`/${method}?`))
+
+describe('slack connector meta', () => {
+  it('crawls per member with the channel selection as the only listing cap', () => {
+    expect(slackConnectorMeta.permissionScopedListing).toEqual({ capFieldIds: ['channel'] })
+  })
+})
+
+describe('listDocuments', () => {
+  it('lists configured channels as deferred stubs without reading their history', async () => {
+    const syncContext: Record<string, unknown> = { syncRunId: 'run-1' }
+    const result = await slackConnector.listDocuments(
+      'token',
+      { channel: ['C0GENERAL'] },
+      undefined,
+      syncContext
+    )
+
+    expect(result.hasMore).toBe(false)
+    expect(result.documents).toEqual([
+      expect.objectContaining({
+        externalId: 'C0GENERAL',
+        title: '#general',
+        content: '',
+        contentDeferred: true,
+        estimatedBytes: CONNECTOR_TEXT_DOCUMENT_MAX_BYTES,
+        contentHash: 'slack-listing:C0GENERAL:run-1',
+        sourceUrl: 'https://app.slack.com/client/T0TEAM/C0GENERAL',
+        metadata: expect.objectContaining({ channelName: 'general' }),
+      }),
+    ])
+    expect(requested('conversations.history')).toHaveLength(0)
+  })
+
+  it('lists every readable channel when none is configured, paging through the cursor', async () => {
+    listNextCursor = 'page-2'
+    const syncContext: Record<string, unknown> = {
+      syncRunId: 'run-1',
+      ...PER_MEMBER_LISTING_CONTEXT,
+    }
+    const first = await slackConnector.listDocuments(
+      'token',
+      { channel: 0 },
+      undefined,
+      syncContext
+    )
+
+    expect(first.documents.map((doc) => doc.externalId)).toEqual(['C0GENERAL', 'G0PLATFORM'])
+    expect(first).toMatchObject({ hasMore: true, nextCursor: 'page-2' })
+    expect(requested('conversations.list')[0]).toContain('types=public_channel%2Cprivate_channel')
+    expect(requested('conversations.list')[0]).toContain('exclude_archived=true')
+
+    listNextCursor = ''
+    listedChannels = []
+    const second = await slackConnector.listDocuments(
+      'token',
+      { channel: 0 },
+      'page-2',
+      syncContext
+    )
+    expect(second).toEqual({ documents: [], nextCursor: undefined, hasMore: false })
+    expect(requested('conversations.list')[1]).toContain('cursor=page-2')
+  })
+
+  it('gives every member of one run the same stub for a channel', async () => {
+    const ada = await slackConnector.listDocuments('ada', {}, undefined, { syncRunId: 'run-7' })
+    const bob = await slackConnector.listDocuments('bob', {}, undefined, { syncRunId: 'run-7' })
+    expect(ada.documents[0].contentHash).toBe(bob.documents[0].contentHash)
+  })
+
+  it('changes the stub between runs so each run re-reads the channel', async () => {
+    const first = await slackConnector.listDocuments('token', {}, undefined, {})
+    const second = await slackConnector.listDocuments('token', {}, undefined, {})
+    expect(first.documents[0].contentHash).not.toBe(second.documents[0].contentHash)
+  })
+})
+
+describe('getDocument', () => {
+  it('builds the transcript under a header with the real content hash', async () => {
+    const doc = await slackConnector.getDocument('token', {}, 'C0GENERAL', {})
+
+    expect(doc).toMatchObject({
+      externalId: 'C0GENERAL',
+      title: '#general',
+      contentHash: 'slack-v3:C0GENERAL:1700000000.000100:1700000200.000100:3:noedit:noreply:0',
+      metadata: expect.objectContaining({ channelName: 'general', messageCount: 2 }),
+    })
+    expect(doc?.content).toBe(
+      [
+        'Channel: #general',
+        'Topic: Company-wide announcements',
+        '',
+        '[2023-11-14T22:15:00.000Z] Person U1: Morning',
+        '[2023-11-14T22:16:40.000Z] Person U2: Shipping today',
+      ].join('\n')
+    )
+  })
+
+  it('keeps a channel with no messages as a live document', async () => {
+    history = []
+    const doc = await slackConnector.getDocument('token', {}, 'C0GENERAL', {})
+    expect(doc?.content).toBe('Channel: #general\nTopic: Company-wide announcements\n')
+    expect(doc?.metadata?.messageCount).toBe(0)
+  })
+
+  it('returns null only for a channel Slack no longer knows', async () => {
+    channelMissing = true
+    await expect(slackConnector.getDocument('token', {}, 'C0GONE', {})).resolves.toBeNull()
+  })
+})