diff --git a/apps/sim/lib/knowledge/__integration__/gitlab-federated.integration.ts b/apps/sim/lib/knowledge/__integration__/gitlab-federated.integration.ts index b217bc67d93..183e4328089 100644 --- a/apps/sim/lib/knowledge/__integration__/gitlab-federated.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/gitlab-federated.integration.ts @@ -25,18 +25,19 @@ import { generateId } from '@sim/utils/id' import { isPlainRecord } from '@sim/utils/object' import { and, eq, inArray } from 'drizzle-orm' import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { resourceScopeFromOwner, resourceScopeKey } from '@/lib/core/resource-scope' import { seedKnowledgeAclFixture } from '@/lib/knowledge/__integration__/seed-source-access-fixture' import { createKnowledgeConnector, updateKnowledgeConnector, } from '@/lib/knowledge/application/connectors' import { refreshConnectorDirectory } from '@/lib/knowledge/connectors/external-group-sync' +import { readLiveDocument, searchLiveKnowledge } from '@/lib/sim-search/live/application' import { decodeLiveReference, encodeLiveReference, - readLiveDocument, - searchLiveKnowledge, -} from '@/lib/sim-search/live/application' + liveReferenceOwner, +} from '@/lib/sim-search/live/reference' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' interface FixturePerson { @@ -133,7 +134,11 @@ describe.skipIf(!fixtureFile)('federated self-hosted GitLab Search', () => { } function documentForReader(documentId: string, person: FixturePerson) { - return encodeLiveReference({ ...decodeLiveReference(documentId), user: person.simId }) + const scope = resourceScopeKey(resourceScopeFromOwner({ organizationId: ids.organizationId })) + return encodeLiveReference( + decodeLiveReference(documentId), + liveReferenceOwner(person.simId, scope) + ) } beforeAll(async () => { diff --git a/apps/sim/lib/knowledge/search/citation.ts b/apps/sim/lib/knowledge/search/citation.ts index c6eb6489991..e17bb6f8cf4 100644 --- a/apps/sim/lib/knowledge/search/citation.ts +++ b/apps/sim/lib/knowledge/search/citation.ts @@ -1,6 +1,12 @@ import { sha256Hex } from '@sim/security/hash' +import { liveDocumentKey } from '@/lib/sim-search/live/reference' -/** Stable opaque citation IDs keep the model from rewriting long live references. */ +/** + * Stable opaque citation IDs keep the model from rewriting long live references. The hash covers + * the document's identity rather than the ID's spelling, so an earlier-form ID and a current one + * for the same document cite as one source. Saved chats keep the citation IDs their tool results + * stored, so changing what is hashed never strands an existing citation. + */ export function liveCitationId(documentId: string): string { - return `live:${sha256Hex(documentId).slice(0, 32)}` + return `live:${sha256Hex(liveDocumentKey(documentId)).slice(0, 32)}` } diff --git a/apps/sim/lib/knowledge/search/filters.test.ts b/apps/sim/lib/knowledge/search/filters.test.ts index d4fb58dc48b..4aa4c73de0e 100644 --- a/apps/sim/lib/knowledge/search/filters.test.ts +++ b/apps/sim/lib/knowledge/search/filters.test.ts @@ -1,5 +1,20 @@ import { describe, expect, it } from 'vitest' +import { liveCitationId } from '@/lib/knowledge/search/citation' import { intersectWorkspaceSearchFilters } from '@/lib/knowledge/search/filters' +import { encodeLiveReference, liveReferenceOwner } from '@/lib/sim-search/live/reference' + +const owner = liveReferenceOwner('reader', 'workspace:w') +const fields = { + account: 'account', + provider: 'slack' as const, + id: '1730912345.123456', + container: 'C1', +} +/** One Slack message in the current ID form and in the earlier form older conversations hold. */ +const currentId = encodeLiveReference(fields, owner) +const earlierId = `live:${Buffer.from( + JSON.stringify({ v: 1, user: 'reader', scope: 'workspace:w', ...fields }) +).toString('base64url')}` describe('Assistant search scope', () => { it('keeps the narrower end of a date window and refuses an empty one', () => { @@ -28,4 +43,22 @@ describe('Assistant search scope', () => { intersectWorkspaceSearchFilters({ documentIds: ['a'] }, { documentIds: ['b'] }) ).toThrow('outside this search') }) + + it('keeps a requested document the user selected in the other ID form', () => { + expect( + intersectWorkspaceSearchFilters({ documentIds: [currentId] }, { documentIds: [earlierId] }) + ).toEqual({ documentIds: [currentId] }) + expect( + intersectWorkspaceSearchFilters({ documentIds: [earlierId] }, { documentIds: [currentId] }) + ).toEqual({ documentIds: [earlierId] }) + }) +}) + +describe('live citation IDs', () => { + it('cite one document as one source whichever ID form named it', () => { + expect(liveCitationId(earlierId)).toBe(liveCitationId(currentId)) + expect(liveCitationId(currentId)).not.toBe( + liveCitationId(encodeLiveReference({ ...fields, id: '1730912345.123457' }, owner)) + ) + }) }) diff --git a/apps/sim/lib/knowledge/search/filters.ts b/apps/sim/lib/knowledge/search/filters.ts index 9467df8a518..1d6e2abb6c3 100644 --- a/apps/sim/lib/knowledge/search/filters.ts +++ b/apps/sim/lib/knowledge/search/filters.ts @@ -1,4 +1,5 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' +import { includesLiveDocument } from '@/lib/sim-search/live/reference' /** Document constraints shared by Search and Assistant, applied before ranking. */ export interface WorkspaceSearchFilters { @@ -19,10 +20,11 @@ export function intersectWorkspaceSearchFilters( if (scope.source && requested.source && scope.source !== requested.source) { throw new OrchestrationError('validation', 'The requested source is outside this search') } - const documentIds = scope.documentIds + const scopeDocumentIds = scope.documentIds + const documentIds = scopeDocumentIds ? requested.documentIds - ? requested.documentIds.filter((id) => scope.documentIds!.includes(id)) - : scope.documentIds + ? requested.documentIds.filter((id) => includesLiveDocument(scopeDocumentIds, id)) + : scopeDocumentIds : requested.documentIds if (documentIds?.length === 0) { throw new OrchestrationError('validation', 'The requested document is outside this search') diff --git a/apps/sim/lib/sim-search/live/application.test.ts b/apps/sim/lib/sim-search/live/application.test.ts index 5bed7b39f05..ca6f8a3d0bf 100644 --- a/apps/sim/lib/sim-search/live/application.test.ts +++ b/apps/sim/lib/sim-search/live/application.test.ts @@ -76,10 +76,9 @@ vi.mock('@/executor/utils/resolved-secret-content-projection', () => ({ projectResolvedSecretModelContent: (value: string) => ({ safe: true, value }), })) +import { resourceScopeFromOwner, resourceScopeKey } from '@/lib/core/resource-scope' import { withSearchDiagnostics } from '@/lib/knowledge/search/diagnostics' import { - decodeLiveReference, - encodeLiveReference, matchesLiveFilters, readLiveDocument, searchLiveKnowledge, @@ -89,6 +88,11 @@ import { NativeSearchError } from '@/lib/sim-search/live/http' import { createPolicyVerifier } from '@/lib/sim-search/live/policy' import { defaultLiveSearchPolicy } from '@/lib/sim-search/live/policy-schema' import { livePolicyFor } from '@/lib/sim-search/live/policy-store' +import { + decodeLiveReference, + encodeLiveReference, + liveReferenceOwner, +} from '@/lib/sim-search/live/reference' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' workspaceAuthzMockFns.mockPermissionSatisfies.mockImplementation( @@ -1185,22 +1189,117 @@ describe('authorized live retrieval', () => { }) ).rejects.toBe('user_stop:test') }) - it('rejects cross-user document references before token resolution', async () => { - const search = await searchLiveKnowledge.execute({ principal, input }) - const reference = decodeLiveReference(search.results[0].documentId) - mocks.resolveAccount.mockClear() - await expect( + describe('live document IDs', () => { + const workspaceScope = resourceScopeKey(resourceScopeFromOwner({ workspaceId: 'workspace' })) + /** The earlier wire form, spelled out so these tests pin what older conversations hold. */ + const legacyId = (fields: Record) => + `live:${Buffer.from(JSON.stringify({ v: 1, ...fields })).toString('base64url')}` + const readerOwner = liveReferenceOwner('reader', workspaceScope) + const fields = { account: 'account', provider: 'google_drive' as const, id: 'doc' } + const read = (documentId: string, extra: Record = {}, as = principal) => readLiveDocument.execute({ - principal, + principal: as, input: { workspaceId: 'workspace', - documentId: encodeLiveReference({ ...reference, user: 'someone-else' }), + documentId, limit: 3, resultSecretRegistry: new ResolvedSecretTraceRegistry([]), + ...extra, }, }) - ).rejects.toThrow('Document not found') - expect(mocks.resolveAccount).not.toHaveBeenCalled() + + it('mints a compact ID naming no reader or scope that reads back', async () => { + const search = await searchLiveKnowledge.execute({ principal, input }) + const [first] = search.results + if (!first) throw new Error('Expected a search result') + const { documentId } = first + expect(documentId).toBe(encodeLiveReference(fields, readerOwner)) + expect(documentId).toMatch(/^live2:google_drive~account~doc~[0-9a-f]{6}$/) + const result = await read(documentId) + expect(result.documentId).toBe(documentId) + expect(result.chunks[0]?.content).toBe('Evidence') + expect(mocks.resolveAccount).toHaveBeenLastCalledWith( + expect.objectContaining({ workspaceId: 'workspace' }), + 'reader', + 'account', + expect.anything() + ) + }) + it("refuses another member's personal account through the reader's own account list", async () => { + mocks.resolveAccount.mockImplementation(async (_owner, _userId, accountId: string) => { + if (accountId !== 'account') + throw new NativeSearchError('reconnect', 'This search connection is no longer available.') + return { account, accessToken: 'secret' } + }) + const theirs = encodeLiveReference({ ...fields, account: 'their-account' }, readerOwner) + await expect(read(theirs)).rejects.toMatchObject({ code: 'unauthorized' }) + expect(mocks.resolveAccount).toHaveBeenCalledWith( + expect.anything(), + 'reader', + 'their-account', + expect.anything() + ) + expect(mocks.read).not.toHaveBeenCalled() + }) + it.each([ + ['another member', 'other-reader', workspaceScope], + [ + 'another scope', + 'reader', + resourceScopeKey(resourceScopeFromOwner({ workspaceId: 'other' })), + ], + ])( + 'refuses an ID minted for %s as not found before resolving any account', + async (_case, user, scope) => { + const theirs = encodeLiveReference(fields, liveReferenceOwner(user, scope)) + await expect(read(theirs)).rejects.toThrow('Document not found in this search scope') + expect(mocks.resolveAccount).not.toHaveBeenCalled() + expect(mocks.read).not.toHaveBeenCalled() + } + ) + it.each([ + ['a missing field', 'live2:google_drive~account'], + ['an unknown provider', 'live2:nowhere~account~doc'], + ['a broken escape', 'live2:google_drive~account~%E0%A4'], + ['too many fields', 'live2:google_drive~a~b~c~d~e~f~g~h'], + ['an unreadable earlier form', 'live:not-a-reference'], + ['no prefix', 'google_drive~account~doc'], + ])('asks the caller to search again for %s', async (_case, documentId) => { + await expect(read(documentId)).rejects.toThrow('Search again') + expect(mocks.resolveAccount).not.toHaveBeenCalled() + }) + it('still reads an earlier-form ID minted for this reader and scope', async () => { + const result = await read(legacyId({ user: 'reader', scope: workspaceScope, ...fields })) + expect(result.chunks[0]?.content).toBe('Evidence') + }) + it.each([ + ['reader', { user: 'someone-else', scope: workspaceScope }], + ['scope', { user: 'reader', scope: 'workspace:elsewhere' }], + ])( + 'rejects an earlier-form ID minted for another %s before token resolution', + async (_case, binding) => { + await expect(read(legacyId({ ...binding, ...fields }))).rejects.toThrow( + 'Document not found' + ) + expect(mocks.resolveAccount).not.toHaveBeenCalled() + } + ) + it('matches a selected document across the earlier and current forms', async () => { + const current = encodeLiveReference(fields, readerOwner) + const earlier = legacyId({ user: 'reader', scope: workspaceScope, ...fields }) + await expect(read(current, { filters: { documentIds: [earlier] } })).resolves.toMatchObject({ + documentId: current, + }) + await expect(read(earlier, { filters: { documentIds: [current] } })).resolves.toBeDefined() + expect( + matchesLiveFilters(document, current, 'google_drive', { documentIds: [earlier] }) + ).toBe(true) + await expect( + read(current, { + filters: { documentIds: [encodeLiveReference({ ...fields, id: 'other' }, readerOwner)] }, + }) + ).rejects.toThrow('outside the selected search filters') + }) }) it('rejects a document moved out of scope while its content was being read', async () => { const search = await searchLiveKnowledge.execute({ principal, input }) diff --git a/apps/sim/lib/sim-search/live/application.ts b/apps/sim/lib/sim-search/live/application.ts index 596cbe372fa..66bcbfc3433 100644 --- a/apps/sim/lib/sim-search/live/application.ts +++ b/apps/sim/lib/sim-search/live/application.ts @@ -11,7 +11,6 @@ import type { } from '@/lib/api/contracts/mothership-assistant-tools' import { type LiveSearchAccountStatus, - liveSearchProviderSchema, type NativeSearchQuery, nativeSearchQueriesSchema, workspaceSearchFiltersSchema, @@ -76,6 +75,13 @@ import { loadLiveSearchPolicies } from '@/lib/sim-search/live/policy-store' import { LIVE_SEARCH_PROVIDER_IDS } from '@/lib/sim-search/live/provider-catalog' import { liveSearchGuidance } from '@/lib/sim-search/live/providers' import { LiveReadError } from '@/lib/sim-search/live/read-error' +import { + encodeLiveReference, + includesLiveDocument, + liveDocumentKey, + liveReferenceOwner, + readLiveReference, +} from '@/lib/sim-search/live/reference' import { queryRelevance } from '@/lib/sim-search/live/text-match' import type { LiveAccount, NativeDocument } from '@/lib/sim-search/live/types' import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-secret-content-projection' @@ -140,43 +146,6 @@ function writeBoundContinuation(provider: BoundProvider, value: BoundContinuatio return cursor } -const referenceSchema = z - .object({ - v: z.literal(1), - user: z.string().min(1), - scope: z.string().min(1), - account: z.string().min(1), - provider: liveSearchProviderSchema, - id: z.string().min(1).max(1000), - container: z.string().max(500).optional(), - kind: z.string().max(200).optional(), - revision: z.string().max(200).optional(), - connectionBinding: z.string().min(1).max(128).optional(), - threadId: z - .string() - .regex(/^\d+\.\d+$/) - .optional(), - }) - .strict() -type Reference = z.output - -export function encodeLiveReference(reference: Reference): string { - const result = `live:${Buffer.from(JSON.stringify(referenceSchema.parse(reference))).toString('base64url')}` - if (result.length > 4000) throw new Error('Provider document reference is too long') - return result -} -export function decodeLiveReference(value: string): Reference { - if (!value.startsWith('live:') || value.length > 4000) - throw new OrchestrationError('not_found', 'Search again to select a live document') - try { - return referenceSchema.parse( - JSON.parse(Buffer.from(value.slice(5), 'base64url').toString('utf8')) - ) - } catch { - throw new OrchestrationError('not_found', 'Invalid live document reference') - } -} - interface LiveSearchOptions { surface?: SearchKnowledgeInput['surface'] query: string @@ -203,7 +172,7 @@ export function matchesLiveFilters( ): boolean { if (!matchesSourceDates(document, provider, filters)) return false if (filters?.source && filters.source !== provider) return false - if (filters?.documentIds && !filters.documentIds.includes(documentId)) return false + if (filters?.documentIds && !includesLiveDocument(filters.documentIds, documentId)) return false if (filters?.modifiedAfter || filters?.modifiedBefore) { const modified = Date.parse(document.modifiedAt ?? '') if (!Number.isFinite(modified)) return false @@ -222,7 +191,7 @@ export function matchesLiveFilters( } return true } -/** A provider result with the reference that binds it to this member, scope, and account. */ +/** A provider result with the ID that locates it within its account. */ interface LiveCandidate { document: NativeDocument documentId: string @@ -231,24 +200,23 @@ interface LiveCandidate { function candidateFor( document: NativeDocument, account: LiveAccount, - owner: ResourceOwner, - userId: string + owner: string ): LiveCandidate { return { document, - documentId: encodeLiveReference({ - v: 1, - user: userId, - scope: resourceScopeKey(resourceScopeFromOwner(owner)), - account: account.id, - provider: account.provider, - id: document.id, - ...(document.container ? { container: document.container } : {}), - ...(document.kind ? { kind: document.kind } : {}), - ...(document.revision ? { revision: document.revision } : {}), - ...(document.connectionBinding ? { connectionBinding: document.connectionBinding } : {}), - ...(document.threadId ? { threadId: document.threadId } : {}), - }), + documentId: encodeLiveReference( + { + account: account.id, + provider: account.provider, + id: document.id, + ...(document.container ? { container: document.container } : {}), + ...(document.kind ? { kind: document.kind } : {}), + ...(document.revision ? { revision: document.revision } : {}), + ...(document.connectionBinding ? { connectionBinding: document.connectionBinding } : {}), + ...(document.threadId ? { threadId: document.threadId } : {}), + }, + owner + ), } } @@ -471,6 +439,7 @@ const liveKnowledgeSearch = defineAuthorizedKnowledgeUseCase({ const userId = requirePrincipalSubjectUserId(principal) if (input.organizationId) await requireOrganizationSearchAvailable(input.organizationId) input.signal?.throwIfAborted() + const owner = liveReferenceOwner(userId, resourceScopeKey(resourceScopeFromOwner(input))) const queries = input.nativeQueries ? nativeSearchQueriesSchema.parse(input.nativeQueries) : undefined @@ -571,7 +540,7 @@ const liveKnowledgeSearch = defineAuthorizedKnowledgeUseCase({ modifiedAfter: instantScopePart(requestedFilters?.modifiedAfter), modifiedBefore: instantScopePart(requestedFilters?.modifiedBefore), documentIds: requestedFilters?.documentIds - ? [...new Set(requestedFilters.documentIds)].sort(compareStrings) + ? [...new Set(requestedFilters.documentIds.map(liveDocumentKey))].sort(compareStrings) : undefined, }) ) @@ -609,7 +578,7 @@ const liveKnowledgeSearch = defineAuthorizedKnowledgeUseCase({ ) const candidates = page.documents .filter((document) => document.id) - .map((document) => candidateFor(document, account, input, userId)) + .map((document) => candidateFor(document, account, owner)) const matchingOf = (verified: LiveCandidate[]) => firstOfEachDocument( verified.filter(({ document, documentId }) => @@ -1072,15 +1041,14 @@ export const readLiveDocument = defineAuthorizedKnowledgeUseCase({ if (input.organizationId) await requireOrganizationSearchAvailable(input.organizationId) if (!Number.isInteger(input.limit) || input.limit < 1 || input.limit > 8) throw new OrchestrationError('validation', 'Invalid document read limit') - const reference = decodeLiveReference(input.documentId) - if ( - reference.user !== userId || - reference.scope !== resourceScopeKey(resourceScopeFromOwner(input)) + const reference = readLiveReference( + input.documentId, + liveReferenceOwner(userId, resourceScopeKey(resourceScopeFromOwner(input))) ) - throw new OrchestrationError('not_found', 'Document not found in this search scope') if ( (input.filters?.source && input.filters.source !== reference.provider) || - (input.filters?.documentIds && !input.filters.documentIds.includes(input.documentId)) + (input.filters?.documentIds && + !includesLiveDocument(input.filters.documentIds, input.documentId)) ) throw new OrchestrationError('not_found', 'Document is outside the selected search filters') /** Cancellation by the caller passes through unclassified. */ diff --git a/apps/sim/lib/sim-search/live/reference.test.ts b/apps/sim/lib/sim-search/live/reference.test.ts new file mode 100644 index 00000000000..86c36a8f6b8 --- /dev/null +++ b/apps/sim/lib/sim-search/live/reference.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from 'vitest' +import { + encodeLiveReference, + type LiveReference, + liveReferenceOwner, + readLiveReference, +} from '@/lib/sim-search/live/reference' + +const owner = liveReferenceOwner('reader', 'workspace:w') + +const thread: LiveReference = { + provider: 'slack', + account: 'account', + id: '1730912345.123456', + container: 'C1', + threadId: '1730900000.000100', +} + +const earlierForm = (reference: LiveReference) => + `live:${Buffer.from( + JSON.stringify({ v: 1, user: 'reader', scope: 'workspace:w', ...reference }) + ).toString('base64url')}` + +describe('live document IDs', () => { + it.each([ + ['dropping its last character', (id: string) => id.slice(0, -1)], + ['dropping the thread it names', (id: string) => id.replace(/~1730900000\.000100(?=~)/, '')], + ['changing the message it names', (id: string) => id.replace('123456', '123457')], + ])('asks for a new search after %s rather than naming another document', (_case, damage) => { + expect(() => readLiveReference(damage(encodeLiveReference(thread, owner)), owner)).toThrow( + 'Search again' + ) + }) + + it.each([ + ['another member', liveReferenceOwner('someone-else', 'workspace:w')], + ['another scope', liveReferenceOwner('reader', 'workspace:other')], + ])('refuses an ID read by %s as not found in this scope', (_case, reader) => { + expect(() => readLiveReference(encodeLiveReference(thread, owner), reader)).toThrow( + 'Document not found in this search scope' + ) + expect(() => readLiveReference(earlierForm(thread), reader)).toThrow( + 'Document not found in this search scope' + ) + }) + + it.each([ + ['the separator and escape character', 'a~b%c%41'], + ['control characters', 'line\none\ttab'], + ['Unicode text', 'docs/設計/仕様書.md'], + ['an unpaired surrogate', 'name\ud800end'], + ])('reads back an ID holding %s', (_case, id) => { + const reference = { provider: 'gitlab', account: 'account', id } as const + expect(readLiveReference(encodeLiveReference(reference, owner), owner)).toEqual(reference) + }) + + it('reads an earlier-form ID minted for the same reader and scope', () => { + expect(readLiveReference(earlierForm(thread), owner)).toEqual(thread) + }) + + it('is never longer than the earlier form for the same document', () => { + const reference = { + provider: 'gitlab', + account: 'account', + id: `src/${'設計書'.repeat(150)}.md`, + } as const + expect(encodeLiveReference(reference, owner).length).toBeLessThan(earlierForm(reference).length) + }) +}) diff --git a/apps/sim/lib/sim-search/live/reference.ts b/apps/sim/lib/sim-search/live/reference.ts new file mode 100644 index 00000000000..2266c55ec3f --- /dev/null +++ b/apps/sim/lib/sim-search/live/reference.ts @@ -0,0 +1,230 @@ +import { sha256Hex } from '@sim/security/hash' +import { z } from 'zod' +import { liveSearchProviderSchema } from '@/lib/api/contracts/mothership-assistant-tools' +import { OrchestrationError } from '@/lib/core/orchestration/types' + +/** + * Live document IDs name a provider document within one of the reader's connected accounts. + * + * They are not an authorization boundary: every read re-resolves the account among the reader's + * own accounts in the request's scope, then re-verifies the document with the provider and the + * admin's source policy. + * + * The current form is `live2:` followed by the document's `~`-separated location fields, trailing + * empty fields dropped, then a checksum: `live2:slack~~~~~~`. + * Clients copy IDs verbatim into later calls, so the form stays readable (provider IDs and UUIDs + * tokenize far better than base64 of them) and carries nothing a model must copy that does not + * locate the document. The checksum covers the location and the reader and scope the ID was minted + * for, so a copy that was truncated or altered, or one read by another member or in another scope, + * is refused as not found here rather than naming a different document or reaching account + * resolution. Earlier IDs (`live:` + base64url JSON naming the reader and scope) still read, with + * their reader and scope checked as before. + */ +const LIVE_V1_PREFIX = 'live:' +const LIVE_V2_PREFIX = 'live2:' +const MAX_REFERENCE_LENGTH = 4000 +const FIELD_SEPARATOR = '~' +const CHECKSUM_LENGTH = 6 + +const fieldsSchema = z + .object({ + provider: liveSearchProviderSchema, + account: z.string().min(1).max(500), + id: z.string().min(1).max(1000), + container: z.string().min(1).max(500).optional(), + kind: z.string().min(1).max(200).optional(), + threadId: z + .string() + .regex(/^\d+\.\d+$/) + .optional(), + revision: z.string().min(1).max(200).optional(), + connectionBinding: z.string().min(1).max(128).optional(), + }) + .strict() + +/** Field order of the v2 form; optional fields last so absent ones cost nothing. */ +const FIELD_ORDER = [ + 'provider', + 'account', + 'id', + 'container', + 'kind', + 'threadId', + 'revision', + 'connectionBinding', +] as const satisfies readonly (keyof z.input)[] + +const legacySchema = z + .object({ + v: z.literal(1), + user: z.string().min(1), + scope: z.string().min(1), + account: z.string().min(1), + provider: liveSearchProviderSchema, + id: z.string().min(1).max(1000), + container: z.string().max(500).optional(), + kind: z.string().max(200).optional(), + revision: z.string().max(200).optional(), + connectionBinding: z.string().min(1).max(128).optional(), + threadId: z + .string() + .regex(/^\d+\.\d+$/) + .optional(), + }) + .strict() + +export type LiveReference = z.output + +/** The reader and scope an ID is minted for and read in. */ +export function liveReferenceOwner(userId: string, scope: string): string { + return `${userId}\u0000${scope}` +} + +/** + * Only the separator, the escape character and control characters are escaped, each as `%XX`; + * every other character, Unicode included, is kept as written, so an ID is never longer than the + * text it names and any string encodes. + */ +function escapeField(value: string): string { + let escaped = '' + for (const character of value) { + const code = character.charCodeAt(0) + escaped += + character === '%' || character === FIELD_SEPARATOR || code < 0x20 || code === 0x7f + ? `%${code.toString(16).toUpperCase().padStart(2, '0')}` + : character + } + return escaped +} + +function unescapeField(value: string): string { + return value.replace(/%([0-9A-F]{2})/g, (_, code: string) => + String.fromCharCode(Number.parseInt(code, 16)) + ) +} + +function locationOf(reference: LiveReference): string { + const fields = fieldsSchema.parse(reference) + const parts = FIELD_ORDER.map((name) => { + const value = fields[name] + return value ? escapeField(value) : '' + }) + while (parts.length > 0 && parts[parts.length - 1] === '') parts.pop() + return parts.join(FIELD_SEPARATOR) +} + +function checksum(owner: string, location: string): string { + return sha256Hex(`${owner}\u0000${location}`).slice(0, CHECKSUM_LENGTH) +} + +/** Mints the current, compact ID for a provider document, for the reader and scope `owner` names. */ +export function encodeLiveReference(reference: LiveReference, owner: string): string { + const location = locationOf(reference) + const result = `${LIVE_V2_PREFIX}${location}${FIELD_SEPARATOR}${checksum(owner, location)}` + if (result.length > MAX_REFERENCE_LENGTH) + throw new Error('Provider document reference is too long') + return result +} + +interface ParsedReference { + reference: LiveReference + /** Whether the ID was minted for the reader and scope `owner` names. */ + mintedFor: (owner: string) => boolean +} + +function parseV2(value: string): ParsedReference { + const separator = value.lastIndexOf(FIELD_SEPARATOR) + if (separator < 0) throw new Error('Missing checksum') + const location = value.slice(0, separator) + const check = value.slice(separator + 1) + const parts = location.split(FIELD_SEPARATOR) + if (parts.length < 3 || parts.length > FIELD_ORDER.length) throw new Error('Invalid field count') + const fields: Record = {} + FIELD_ORDER.forEach((name, index) => { + const part = parts[index] + if (part) fields[name] = unescapeField(part) + }) + return { + reference: fieldsSchema.parse(fields), + mintedFor: (owner) => check === checksum(owner, location), + } +} + +function parseV1(body: string): ParsedReference { + const { v, user, scope, ...rest } = legacySchema.parse( + JSON.parse(Buffer.from(body, 'base64url').toString('utf8')) + ) + const fields = Object.fromEntries( + Object.entries(rest).filter(([, value]) => value !== undefined && value !== '') + ) + return { + reference: fieldsSchema.parse(fields), + mintedFor: (owner) => owner === liveReferenceOwner(user, scope), + } +} + +const SEARCH_AGAIN = 'Search again to select a live document' + +function parse(value: string): ParsedReference { + try { + if (value.length <= MAX_REFERENCE_LENGTH) { + if (value.startsWith(LIVE_V2_PREFIX)) return parseV2(value.slice(LIVE_V2_PREFIX.length)) + if (value.startsWith(LIVE_V1_PREFIX)) return parseV1(value.slice(LIVE_V1_PREFIX.length)) + } + } catch {} + throw new OrchestrationError('not_found', `Invalid live document reference. ${SEARCH_AGAIN}`) +} + +/** + * The document a current or earlier live ID names, without checking whom it was minted for. Reads + * use {@link readLiveReference}; anything that is not a live ID asks the caller to search again. + */ +export function decodeLiveReference(value: string): LiveReference { + return parse(value).reference +} + +/** + * The document an ID names, when it was minted for the reader and scope `owner` names. A current ID + * whose checksum does not hold for them, because it was minted for someone or somewhere else or + * was altered, is not found in this scope. + */ +export function readLiveReference(value: string, owner: string): LiveReference { + const { reference, mintedFor } = parse(value) + if (!mintedFor(owner)) + throw new OrchestrationError( + 'not_found', + value.startsWith(LIVE_V2_PREFIX) + ? `Document not found in this search scope. ${SEARCH_AGAIN}` + : 'Document not found in this search scope' + ) + return reference +} + +/** + * The identity two document IDs share when they name the same document: its location, whichever + * form the ID arrived in, or the value itself for any other ID. Compare document IDs through this, + * never as raw strings, so a v1 ID from an earlier conversation matches the v2 ID a new search + * returns for the same document. + */ +export function liveDocumentKey(documentId: string): string { + if (!documentId.startsWith(LIVE_V1_PREFIX) && !documentId.startsWith(LIVE_V2_PREFIX)) + return documentId + try { + return `${LIVE_V2_PREFIX}${locationOf(decodeLiveReference(documentId))}` + } catch { + return documentId + } +} + +/** Each filter list's keys, decoded once however many results a search checks against it. */ +const listKeys = new WeakMap>() + +/** Whether `documentId` names one of `documentIds`, comparing by {@link liveDocumentKey}. */ +export function includesLiveDocument(documentIds: readonly string[], documentId: string): boolean { + let keys = listKeys.get(documentIds) + if (!keys) { + keys = new Set(documentIds.map(liveDocumentKey)) + listKeys.set(documentIds, keys) + } + return keys.has(liveDocumentKey(documentId)) +} diff --git a/scripts/check-explicit-any.baseline.json b/scripts/check-explicit-any.baseline.json index 8c3029c65c2..a4fb7f4bbf0 100644 --- a/scripts/check-explicit-any.baseline.json +++ b/scripts/check-explicit-any.baseline.json @@ -1511,7 +1511,6 @@ "apps/sim/lib/knowledge/read-access.test.ts": 1, "apps/sim/lib/knowledge/read-access.ts": 1, "apps/sim/lib/knowledge/search/budget.test.ts": 1, - "apps/sim/lib/knowledge/search/filters.ts": 1, "apps/sim/lib/knowledge/search/queries.test.ts": 10, "apps/sim/lib/knowledge/search/queries.ts": 3, "apps/sim/lib/knowledge/tags/filter-resolution.ts": 1,