diff --git a/.changeset/hybrid-ai-search-retrieval.md b/.changeset/hybrid-ai-search-retrieval.md new file mode 100644 index 0000000000000..2c914e0cc7bc2 --- /dev/null +++ b/.changeset/hybrid-ai-search-retrieval.md @@ -0,0 +1,9 @@ +--- +'@rocket.chat/ai-search': minor +'@rocket.chat/core-services': minor +'@rocket.chat/rest-typings': minor +'@rocket.chat/i18n': minor +'@rocket.chat/meteor': minor +--- + +Adds hybrid retrieval to AI Search. A single search balance setting decides how much semantic retrieval contributes relative to keyword search, so a workspace can find messages by meaning without losing exact matches on error codes, ticket ids or function names. An optional recency boost, disabled by default, promotes newer messages after relevance ranking. diff --git a/apps/meteor/server/api/v1/ai-search.ts b/apps/meteor/server/api/v1/ai-search.ts index 39290d189affd..58e50517c01af 100644 --- a/apps/meteor/server/api/v1/ai-search.ts +++ b/apps/meteor/server/api/v1/ai-search.ts @@ -132,6 +132,13 @@ const parseCommaList = (value: string | undefined): string[] => { }; const parseQueryDate = (value: string | undefined): Date | undefined => (value ? new Date(value) : undefined); +const parseSearchType = (value: string | undefined): 'semantic' | 'keyword' | 'hybrid' | undefined => { + if (value === 'semantic' || value === 'keyword' || value === 'hybrid') { + return value; + } + + return undefined; +}; const getRoomMap = async (roomIds: string[]): Promise>> => { if (!roomIds.length) { @@ -254,6 +261,7 @@ API.v1.get( const fromUsernames = parseCommaList(this.queryParams.fromUsernames); const startDate = parseQueryDate(this.queryParams.startDate); const endDate = parseQueryDate(this.queryParams.endDate); + const searchType = parseSearchType(this.queryParams.searchType); const aiSearchStatus = await AISearch.status().catch((error) => { this.logger.warn({ msg: 'AI search status unavailable', err: error }); @@ -283,6 +291,7 @@ API.v1.get( startDate: startDate?.toISOString(), endDate: endDate?.toISOString(), }, + searchType, limit: intelligentLimit, }); } catch (error) { diff --git a/apps/meteor/server/services/ai-search/service.ts b/apps/meteor/server/services/ai-search/service.ts index 67cb473dd26c4..0959173c07a12 100644 --- a/apps/meteor/server/services/ai-search/service.ts +++ b/apps/meteor/server/services/ai-search/service.ts @@ -1,14 +1,27 @@ import { AI_LICENSE_MODULE, AI_SEARCH_PAGE_SIZE, + applyTemporalRerank, buildIntelligentSearchPipelineFilters, + DEFAULT_INTELLIGENT_SEARCH_RECENCY_HALF_LIFE_DAYS, + DEFAULT_INTELLIGENT_SEARCH_SEMANTIC_WEIGHT, + filterSemanticCandidatesByMinimumSimilarity, + fuseCandidatesWithWeightedRRF, generateOpenAICompatibleSearchAnswer, + INTELLIGENT_SEARCH_CANDIDATE_MULTIPLIER, listOpenAICompatibleModels, + MAX_INTELLIGENT_SEARCH_CANDIDATES, MAX_SEARCH_ANSWER_MESSAGES, MAX_SEARCH_ANSWER_TEXT_LENGTH, MAX_SEARCH_FILTER_VALUES, + MIN_INTELLIGENT_SEARCH_CANDIDATES, normalizeIntelligentSearchCandidates, + toRankedCandidates, + type FusedIntelligentSearchCandidate, + type IntelligentSearchCandidate, + type IntelligentSearchType, searchIntelligentPipeline, + type IntelligentSearchPipelineFilters, type IntelligentSearchFilters, type IntelligentSearchPipelineConfig, type OpenAICompatibleProviderConfig, @@ -154,6 +167,127 @@ export class AISearchService extends ServiceClass implements IAISearchService { }; } + // `hybrid` defers to the configured balance rather than forcing a mid value, so a workspace pinned to + // one retriever stays pinned. + private resolveSemanticWeight(searchType: IntelligentSearchType | undefined): number { + if (searchType === 'keyword') { + return 0; + } + + if (searchType === 'semantic') { + return 100; + } + + const configuredWeight = Number(settings.get('AI_Intelligent_Search_Semantic_Weight')); + if (!Number.isFinite(configuredWeight)) { + return DEFAULT_INTELLIGENT_SEARCH_SEMANTIC_WEIGHT; + } + + return Math.min(100, Math.max(0, Math.floor(configuredWeight))); + } + + private getRecencyWeight(): number { + const configuredWeight = Number(settings.get('AI_Intelligent_Search_Recency_Weight')); + if (!Number.isFinite(configuredWeight)) { + return 0; + } + + return Math.min(100, Math.max(0, Math.floor(configuredWeight))); + } + + private async queryPipelineCandidates({ + query, + config, + classifications, + pipelineFilters, + limit, + sourceMode, + }: { + query: string; + config: IntelligentSearchPipelineConfig; + classifications: string[]; + pipelineFilters: IntelligentSearchPipelineFilters; + limit: number; + sourceMode: 'semantic' | 'keyword'; + }): Promise { + const raw = await searchIntelligentPipeline({ + query, + config, + classifications, + pipelineFilters, + limit, + fetch: fetchWithSsrfValidation, + logger, + mode: sourceMode, + }); + + return normalizeIntelligentSearchCandidates(raw, [], limit, logger, sourceMode); + } + + private async buildSearchCandidatesForMode( + query: string, + config: IntelligentSearchPipelineConfig, + classifications: string[], + pipelineFilters: IntelligentSearchPipelineFilters, + limit: number, + semanticWeight: number, + ): Promise { + const candidateLimit = this.getSearchCandidateLimit(limit); + const queryBranch = (sourceMode: 'semantic' | 'keyword') => + this.queryPipelineCandidates({ query, config, classifications, pipelineFilters, limit: candidateLimit, sourceMode }); + + const minimumSimilarityPercent = Number(config.minimumSimilarityPercent || 0); + + // at the extremes the other retriever is never requested + if (semanticWeight === 0) { + return toRankedCandidates(await queryBranch('keyword')); + } + + if (semanticWeight === 100) { + return toRankedCandidates(filterSemanticCandidatesByMinimumSimilarity(await queryBranch('semantic'), minimumSimilarityPercent)); + } + + // allSettled, not all: searchIntelligentPipeline rethrows on network failure and timeout, and one + // flaky branch must degrade hybrid to the survivor rather than to an empty result set + const [semanticResult, keywordResult] = await Promise.allSettled([queryBranch('semantic'), queryBranch('keyword')]); + const keywordCandidates = keywordResult.status === 'fulfilled' ? keywordResult.value : undefined; + const semanticCandidates = + semanticResult.status === 'fulfilled' + ? filterSemanticCandidatesByMinimumSimilarity(semanticResult.value, minimumSimilarityPercent) + : undefined; + + if (!semanticCandidates) { + if (!keywordCandidates) { + throw semanticResult.status === 'rejected' ? semanticResult.reason : new Error('error-ai-search-retrieval-failed'); + } + + logger.warn({ msg: 'Intelligent search branch failed, serving the surviving retriever', failedBranch: 'semantic' }); + + return toRankedCandidates(keywordCandidates); + } + + if (!keywordCandidates) { + logger.warn({ msg: 'Intelligent search branch failed, serving the surviving retriever', failedBranch: 'keyword' }); + + return toRankedCandidates(semanticCandidates); + } + + // Preserve the union until visibility filtering and temporal reranking have run. + return fuseCandidatesWithWeightedRRF( + semanticCandidates, + keywordCandidates, + semanticWeight, + semanticCandidates.length + keywordCandidates.length, + ); + } + + // Over-fetch to improve fusion overlap and reduce short pages after permission filtering. + private getSearchCandidateLimit(requestedLimit: number): number { + const scaledLimit = Math.max(requestedLimit, AI_SEARCH_PAGE_SIZE) * INTELLIGENT_SEARCH_CANDIDATE_MULTIPLIER; + + return Math.min(MAX_INTELLIGENT_SEARCH_CANDIDATES, Math.max(MIN_INTELLIGENT_SEARCH_CANDIDATES, scaledLimit)); + } + async status(): Promise { const hasIntelligentSearchLicense = await License.hasModule(AI_LICENSE_MODULE); const intelligentSearchEnabled = settings.get('AI_Intelligent_Search_Enabled'); @@ -217,13 +351,14 @@ export class AISearchService extends ServiceClass implements IAISearchService { } private async normalizeIntelligentResults( - rawSearchResults: unknown, + searchCandidates: IntelligentSearchCandidate[], userId: string, limit = AI_SEARCH_PAGE_SIZE, ): Promise { - const candidates = normalizeIntelligentSearchCandidates(rawSearchResults, [], limit, logger); + // the whole pool is resolved, not just the first page: pre-slicing here returns short pages once + // permission filtering below drops a candidate const msgIdSet = new Set(); - for (const { msgId } of candidates) { + for (const { msgId } of searchCandidates) { if (msgId) { msgIdSet.add(msgId); } @@ -249,7 +384,7 @@ export class AISearchService extends ServiceClass implements IAISearchService { ]); const normalizedResults: AISearchResult[] = []; - for (const result of candidates) { + for (const result of searchCandidates) { // candidates without a visible database message could surface stale pipeline text const dbMessage = result.msgId ? messageMap.get(result.msgId) : undefined; if (!dbMessage) { @@ -315,11 +450,13 @@ export class AISearchService extends ServiceClass implements IAISearchService { userId, filters: rawFilters, limit = AI_SEARCH_PAGE_SIZE, + searchType, }: { query: string; userId: string; filters?: AISearchFilters; limit?: number; + searchType?: IntelligentSearchType; }): Promise { const hasIntelligentSearchLicense = await License.hasModule(AI_LICENSE_MODULE); const intelligentSearchEnabled = settings.get('AI_Intelligent_Search_Enabled'); @@ -365,17 +502,14 @@ export class AISearchService extends ServiceClass implements IAISearchService { return []; } - const json = await searchIntelligentPipeline({ - query, - config, - classifications, - pipelineFilters, - limit, - fetch: fetchWithSsrfValidation, - logger, + const semanticWeight = this.resolveSemanticWeight(searchType); + const candidates = await this.buildSearchCandidatesForMode(query, config, classifications, pipelineFilters, limit, semanticWeight); + const rerankedCandidates = applyTemporalRerank(candidates, { + recencyWeight: this.getRecencyWeight(), + halfLifeDays: DEFAULT_INTELLIGENT_SEARCH_RECENCY_HALF_LIFE_DAYS, }); - return this.normalizeIntelligentResults(json, userId, limit); + return this.normalizeIntelligentResults(rerankedCandidates, userId, limit); } async answer({ query, messages }: { query: string; messages: AISearchAnswerMessage[] }): Promise { diff --git a/apps/meteor/server/settings/ai.ts b/apps/meteor/server/settings/ai.ts index 90470bcdf30fb..ab1a6819c515a 100644 --- a/apps/meteor/server/settings/ai.ts +++ b/apps/meteor/server/settings/ai.ts @@ -52,6 +52,30 @@ export const createAISettings = async (): Promise => { i18nDescription: 'AI_Intelligent_Search_Enabled_Description', }); + await settingsRegistry.add('AI_Intelligent_Search_Semantic_Weight', 50, { + group: AI_SETTINGS_GROUP, + section: 'Intelligent_Search', + type: 'int', + i18nLabel: 'AI_Intelligent_Search_Semantic_Weight', + i18nDescription: 'AI_Intelligent_Search_Semantic_Weight_Description', + enterprise: true, + modules: [AI_LICENSE_MODULE], + invalidValue: 50, + enableQuery: { _id: 'AI_Intelligent_Search_Enabled', value: true }, + }); + + await settingsRegistry.add('AI_Intelligent_Search_Recency_Weight', 0, { + group: AI_SETTINGS_GROUP, + section: 'Intelligent_Search', + type: 'int', + i18nLabel: 'AI_Intelligent_Search_Recency_Weight', + i18nDescription: 'AI_Intelligent_Search_Recency_Weight_Description', + enterprise: true, + modules: [AI_LICENSE_MODULE], + invalidValue: 0, + enableQuery: { _id: 'AI_Intelligent_Search_Enabled', value: true }, + }); + await settingsRegistry.add('AI_Intelligent_Search_Pipeline_Base_URL', '', { group: AI_SETTINGS_GROUP, section: 'Intelligent_Search', diff --git a/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts b/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts index b2ee75ce9fc9d..44248277266ed 100644 --- a/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts +++ b/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts @@ -71,6 +71,8 @@ const cursor = (items: T[]): CursorResult => ({ const settings: Record = { AI_Intelligent_Search_Enabled: true, + AI_Intelligent_Search_Semantic_Weight: 100, + AI_Intelligent_Search_Recency_Weight: 0, AI_Intelligent_Search_Pipeline_Base_URL: 'https://pipeline.example.com', AI_Intelligent_Search_Pipeline_ID: 'workspace', AI_Intelligent_Search_API_Key: 'key', @@ -115,6 +117,7 @@ describe('AISearchService', () => { Subscriptions.findByUserIdAndRoomIds.callsFake((_userId: string, roomIds: string[]) => cursor(roomIds.filter((roomId) => roomId === 'allowed' || roomId === 'room-general').map((rid) => ({ rid }))), ); + Subscriptions.findByUserId.callsFake(() => cursor([{ rid: 'allowed' }])); Messages.findVisibleByIds.callsFake((msgIds: string[]) => cursor( msgIds.map((msgId) => ({ @@ -196,7 +199,8 @@ describe('AISearchService', () => { const [, options] = serverFetch.firstCall.args; const body = JSON.parse(options.body); - expect(body.params.k).to.equal(5); + // the retriever is asked for a candidate pool, not the requested page + expect(body.params.k).to.equal(20); expect(body.filters).to.deep.equal({ room_id: { $in: subscribedRoomIds }, }); @@ -216,6 +220,304 @@ describe('AISearchService', () => { ).to.be.true; }); + it('uses explicit searchType keyword when requested', async () => { + serverFetch.resolves({ + ok: true, + status: 200, + json: async () => ({ + results: [{ metadata: { room_id: 'allowed', msg_id: 'allowed-msg' }, text: 'keyword pipeline text', score: 0.77 }], + }), + text: async () => '', + }); + + await createService().search({ + query: 'fruit', + userId: 'user-id', + limit: 5, + searchType: 'keyword', + }); + + const requestBody = JSON.parse(serverFetch.firstCall.args[1].body); + expect(requestBody.type).to.equal('search'); + expect(requestBody.classification).to.deep.equal({ classifications: ['user', 'admin'], search_type: 1 }); + expect(requestBody.params).to.not.have.property('threshold'); + }); + + it('queries only the keyword retriever when the balance is 0', async () => { + cachedSettings.get.callsFake((key: string) => (key === 'AI_Intelligent_Search_Semantic_Weight' ? 0 : settings[key])); + serverFetch.resolves({ + ok: true, + status: 200, + json: async () => ({ + results: [{ metadata: { room_id: 'allowed', msg_id: 'keyword-msg' }, text: 'keyword text', score: 0.9 }], + }), + text: async () => '', + }); + + await createService().search({ query: 'fruit', userId: 'user-id' }); + + expect(serverFetch.callCount).to.equal(1); + const requestBody = JSON.parse(serverFetch.firstCall.args[1].body); + expect(requestBody.type).to.equal('search'); + }); + + it('queries only the semantic retriever when the balance is 100', async () => { + cachedSettings.get.callsFake((key: string) => (key === 'AI_Intelligent_Search_Semantic_Weight' ? 100 : settings[key])); + serverFetch.resolves({ + ok: true, + status: 200, + json: async () => ({ + results: [{ metadata: { room_id: 'allowed', msg_id: 'semantic-msg' }, text: 'semantic text', score: 0.88 }], + }), + text: async () => '', + }); + + await createService().search({ query: 'fruit', userId: 'user-id' }); + + expect(serverFetch.callCount).to.equal(1); + const requestBody = JSON.parse(serverFetch.firstCall.args[1].body); + expect(requestBody.type).to.equal('similarity'); + expect(requestBody.params).to.have.property('threshold'); + }); + + it('uses weighted hybrid with semantic threshold filtering only on semantic branch', async () => { + cachedSettings.get.callsFake((key: string) => (key === 'AI_Intelligent_Search_Semantic_Weight' ? 50 : settings[key])); + + serverFetch.reset(); + serverFetch + .onCall(0) + .resolves({ + ok: true, + status: 200, + json: async () => ({ + results: [ + { metadata: { room_id: 'allowed', msg_id: 'allowed-msg' }, text: 'semantic good', score: 0.2 }, + { metadata: { room_id: 'allowed', msg_id: 'filtered-msg' }, text: 'semantic filtered', score: 0.49 }, + ], + }), + text: async () => '', + }) + .onCall(1) + .resolves({ + ok: true, + status: 200, + json: async () => ({ + results: [{ metadata: { room_id: 'allowed', msg_id: 'keyword-msg' }, text: 'keyword text', score: 0.4 }], + }), + text: async () => '', + }); + + const results = await createService().search({ query: 'fruit', userId: 'user-id', limit: 5 }); + + expect(serverFetch.callCount).to.equal(2); + expect(results).to.deep.equal([ + { + _id: 'allowed-msg', + rid: 'allowed', + msgId: 'allowed-msg', + text: 'allowed-msg from db', + ts: '2026-01-05T12:00:00.000Z', + u: { username: 'alice', name: 'Alice' }, + score: 0.8, + room: { _id: 'allowed', t: 'c', name: 'general', fname: 'General' }, + }, + { + // keyword-sourced: carries no similarity, because the pipeline's full-text rank is not one + _id: 'keyword-msg', + rid: 'allowed', + msgId: 'keyword-msg', + text: 'keyword-msg from db', + ts: '2026-01-05T12:00:00.000Z', + u: { username: 'alice', name: 'Alice' }, + room: { _id: 'allowed', t: 'c', name: 'general', fname: 'General' }, + }, + ]); + }); + + it('scales the candidate pool with the requested page and caps it', async () => { + serverFetch.resolves({ ok: true, status: 200, json: async () => ({ results: [] }), text: async () => '' }); + + const service = createService(); + await service.search({ query: 'fruit', userId: 'user-id', limit: 9 }); + expect(JSON.parse(serverFetch.lastCall.args[1].body).params.k).to.equal(27); + + // the cap stays above the largest page so permission filtering cannot shorten it + await service.search({ query: 'fruit', userId: 'user-id', limit: 50 }); + expect(JSON.parse(serverFetch.lastCall.args[1].body).params.k).to.equal(100); + }); + + it('keeps a full page of hybrid results when fusion candidates are not visible', async () => { + cachedSettings.get.callsFake((key: string) => (key === 'AI_Intelligent_Search_Semantic_Weight' ? 50 : settings[key])); + // only the last two candidates resolve to a visible message + Messages.findVisibleByIds.callsFake((msgIds: string[]) => + cursor( + msgIds + .filter((msgId) => msgId === 'visible-a' || msgId === 'visible-b') + .map((msgId) => ({ + _id: msgId, + rid: 'allowed', + msg: `${msgId} from db`, + ts: new Date('2026-01-05T12:00:00.000Z'), + u: { username: 'alice', name: 'Alice' }, + })), + ), + ); + const semanticResults = [ + { metadata: { room_id: 'allowed', msg_id: 'hidden-1' }, score: 0.2 }, + { metadata: { room_id: 'allowed', msg_id: 'hidden-2' }, score: 0.25 }, + { metadata: { room_id: 'allowed', msg_id: 'visible-a' }, score: 0.3 }, + ]; + const keywordResults = [{ metadata: { room_id: 'allowed', msg_id: 'visible-b' }, score: 0.4 }]; + serverFetch.reset(); + serverFetch + .onCall(0) + .resolves({ ok: true, status: 200, json: async () => ({ results: semanticResults }), text: async () => '' }) + .onCall(1) + .resolves({ ok: true, status: 200, json: async () => ({ results: keywordResults }), text: async () => '' }); + + const results = await createService().search({ query: 'fruit', userId: 'user-id', limit: 2 }); + + expect(results.map(({ _id }: { _id: string }) => _id)).to.deep.equal(['visible-b', 'visible-a']); + }); + + it('retains the other branch beyond the candidate cap when higher-ranked messages are inaccessible', async () => { + cachedSettings.get.callsFake((key: string) => (key === 'AI_Intelligent_Search_Semantic_Weight' ? 90 : settings[key])); + serverFetch.callsFake(async (_url: string, options: { body: string }) => { + const { type, params } = JSON.parse(options.body); + const results = Array.from({ length: params.k }, (_, index) => ({ + metadata: { room_id: type === 'similarity' ? 'forbidden' : 'allowed', msg_id: `${type}-${index}` }, + score: 0.2, + })); + return { ok: true, status: 200, json: async () => ({ results }), text: async () => '' }; + }); + Messages.findVisibleByIds.callsFake((msgIds: string[]) => + cursor(msgIds.map((_id) => ({ _id, rid: _id.startsWith('similarity-') ? 'forbidden' : 'allowed', msg: _id }))), + ); + + const results = await createService().search({ query: 'fruit', userId: 'user-id', limit: 2 }); + + expect(results.map(({ _id }: { _id: string }) => _id)).to.deep.equal(['search-0', 'search-1']); + }); + + it('promotes fresher messages once the recency boost is enabled', async () => { + const timestamps: Record = { + stale: '2020-01-01T12:00:00.000Z', + fresh: new Date().toISOString(), + }; + Messages.findVisibleByIds.callsFake((msgIds: string[]) => + cursor( + msgIds.map((msgId) => ({ + _id: msgId, + rid: 'allowed', + msg: `${msgId} from db`, + ts: new Date(timestamps[msgId]), + u: { username: 'alice', name: 'Alice' }, + })), + ), + ); + const pipelineResults = { + results: [ + { metadata: { room_id: 'allowed', msg_id: 'stale', timestamp: timestamps.stale }, score: 0.2 }, + { metadata: { room_id: 'allowed', msg_id: 'fresh', timestamp: timestamps.fresh }, score: 0.21 }, + ], + }; + serverFetch.resolves({ ok: true, status: 200, json: async () => pipelineResults, text: async () => '' }); + + const withoutBoost = await createService().search({ query: 'fruit', userId: 'user-id', limit: 5 }); + expect(withoutBoost.map(({ _id }: { _id: string }) => _id)).to.deep.equal(['stale', 'fresh']); + + cachedSettings.get.callsFake((key: string) => (key === 'AI_Intelligent_Search_Recency_Weight' ? 100 : settings[key])); + const withBoost = await createService().search({ query: 'fruit', userId: 'user-id', limit: 5 }); + expect(withBoost.map(({ _id }: { _id: string }) => _id)).to.deep.equal(['fresh', 'stale']); + }); + + it('serves the surviving retriever when one hybrid branch fails', async () => { + cachedSettings.get.callsFake((key: string) => (key === 'AI_Intelligent_Search_Semantic_Weight' ? 50 : settings[key])); + serverFetch.reset(); + serverFetch + .onCall(0) + .resolves({ + ok: true, + status: 200, + json: async () => ({ results: [{ metadata: { room_id: 'allowed', msg_id: 'allowed-msg' }, score: 0.2 }] }), + text: async () => '', + }) + .onCall(1) + .rejects(new Error('keyword branch timed out')); + + const results = await createService().search({ query: 'fruit', userId: 'user-id', limit: 5 }); + + expect(results.map(({ _id }: { _id: string }) => _id)).to.deep.equal(['allowed-msg']); + }); + + it('gives up only when both hybrid branches fail', async () => { + cachedSettings.get.callsFake((key: string) => (key === 'AI_Intelligent_Search_Semantic_Weight' ? 50 : settings[key])); + serverFetch.reset(); + serverFetch.rejects(new Error('pipeline unreachable')); + + await createService() + .search({ query: 'fruit', userId: 'user-id', limit: 5 }) + .then( + () => expect.fail('expected the search to reject'), + (error: Error) => expect(error.message).to.equal('pipeline unreachable'), + ); + }); + + for (const failedType of ['similarity', 'search']) { + it(`serves the surviving retriever when ${failedType} returns HTTP 503`, async () => { + cachedSettings.get.callsFake((key: string) => (key === 'AI_Intelligent_Search_Semantic_Weight' ? 50 : settings[key])); + serverFetch.callsFake(async (_url: string, options: { body: string }) => { + const failed = JSON.parse(options.body).type === failedType; + return { + ok: !failed, + status: failed ? 503 : 200, + json: async () => ({ results: [{ id: 'allowed-msg', score: 0.2 }] }), + text: async () => '', + }; + }); + + const results = await createService().search({ query: 'fruit', userId: 'user-id' }); + + expect(results.map(({ _id }: { _id: string }) => _id)).to.deep.equal(['allowed-msg']); + }); + } + + it('rejects when both retrievers return HTTP errors', async () => { + cachedSettings.get.callsFake((key: string) => (key === 'AI_Intelligent_Search_Semantic_Weight' ? 50 : settings[key])); + serverFetch.resolves({ ok: false, status: 503, text: async () => '' }); + + await createService() + .search({ query: 'fruit', userId: 'user-id' }) + .then( + () => expect.fail('expected the search to reject'), + (error: Error) => expect(error.message).to.equal('Intelligent search pipeline returned HTTP 503'), + ); + }); + + it('lets an explicit searchType pin an endpoint of the balance without an admin change', async () => { + cachedSettings.get.callsFake((key: string) => (key === 'AI_Intelligent_Search_Semantic_Weight' ? 50 : settings[key])); + serverFetch.resolves({ + ok: true, + status: 200, + json: async () => ({ results: [{ metadata: { room_id: 'allowed', msg_id: 'allowed-msg' }, score: 0.2 }] }), + text: async () => '', + }); + + const service = createService(); + await service.search({ query: 'fruit', userId: 'user-id', searchType: 'semantic' }); + expect(serverFetch.callCount).to.equal(1); + expect(JSON.parse(serverFetch.lastCall.args[1].body).type).to.equal('similarity'); + + serverFetch.resetHistory(); + await service.search({ query: 'fruit', userId: 'user-id', searchType: 'keyword' }); + expect(serverFetch.callCount).to.equal(1); + expect(JSON.parse(serverFetch.lastCall.args[1].body).type).to.equal('search'); + + serverFetch.resetHistory(); + await service.search({ query: 'fruit', userId: 'user-id', searchType: 'hybrid' }); + expect(serverFetch.callCount).to.equal(2); + }); + it('resolves room-name filters before querying the pipeline', async () => { Rooms.findOneByNameOrFname.resolves({ _id: 'room-general' }); Subscriptions.findByUserId.returns(cursor([{ rid: 'room-general' }])); diff --git a/docs/features/ai-search-hybrid.md b/docs/features/ai-search-hybrid.md new file mode 100644 index 0000000000000..9e1a1cf9a6591 --- /dev/null +++ b/docs/features/ai-search-hybrid.md @@ -0,0 +1,100 @@ +# AI Search: hybrid retrieval and temporal reranking + +## Retrieval + +Both retrievers are the *same* pipeline endpoint (`POST /pipelines/{id}/search`), distinguished only by +the request body: + +| Retriever | Pipeline request | Threshold sent | +| --- | --- | --- | +| semantic | `type: "similarity"`, `classification.search_type: 2` | yes | +| keyword | `type: "search"`, `classification.search_type: 1` | no | + +Which of them runs is decided by a single setting, `AI_Intelligent_Search_Semantic_Weight` (0-100): + +| Balance | Behaviour | +| --- | --- | +| `0` | keyword only - the semantic retriever is never called | +| `1`-`99` | both in parallel, fused with weighted RRF | +| `100` | semantic only - the keyword retriever is never called | + +There is deliberately no separate "search mode" setting: the balance already expresses every mode, and a +second control would only let the two disagree. A caller can pin an endpoint of the range per request +with the `searchType` query parameter on `GET /v1/ai.search` (`keyword` maps to 0, `semantic` to 100, +`hybrid` to whatever the setting says). + +Fusion happens here rather than in the pipeline because the pipeline's own `type: "hybrid"` placeholder +returns `501 Hybrid placeholder is not implemented yet`, and its parameter schema exposes no weight. + +## Score conventions + +The two retrievers report scores on incompatible scales: + +- semantic `score` is a **cosine distance** - *lower* is better +- keyword `score` is a **full-text rank** - *higher* is better + +`normalizeIntelligentSearchCandidates` converts distance to similarity (`1 - distance`) for display and +for the guardrail, and does so only for semantic candidates. Both retrievers report their number in the +same `score` field, so reading a keyword hit's rank as a distance would invert it and fabricate a +confident similarity. Keyword candidates therefore carry no `score`, and the results UI renders a match +percentage only where one genuinely exists. + +## Weighted RRF + +For a document `d`, with `w = AI_Intelligent_Search_Semantic_Weight / 100` and `C = 60`: + +``` +score(d) = (1 - w) / (C + rank_fulltext(d)) + w / (C + rank_semantic(d)) +``` + +Fusion works on **rank positions only**, which is what makes the incompatible scales a non-problem. A +branch that did not return `d` contributes nothing. + +The two branches are issued with `Promise.allSettled`, not `Promise.all`: if one retriever throws +(HTTP error, network failure, or the pipeline's 10s timeout) the search degrades to the surviving retriever rather +than returning nothing. A double failure rejects the service call. The existing REST handler logs +that error and returns an empty result list. + +`C` and the candidate pool size are implementation parameters and are intentionally not admin settings. + +## The similarity guardrail + +`AI_Intelligent_Search_Min_Similarity_Percent` applies **only to semantic candidates**, through the +pipeline request's distance threshold and again after retrieval. A keyword hit is never discarded for +being semantically unremarkable - that is precisely the +case hybrid search exists to serve (exact error codes, ticket ids, function names). + +It defaults to `0` (disabled) and should stay that way for most workspaces: a fixed embedding threshold +is brittle across embedding models, query length, language and corpus, whereas ranking is stable. Treat +it as a garbage-result guardrail, not a quality control. If the pipeline omits all similarity metadata, +unscored semantic candidates are preserved for compatibility. + +## Temporal reranking + +Applied after fusion, so relevance selects the candidates and freshness only reorders them. With +`w = AI_Intelligent_Search_Recency_Weight / 100`: + +``` +final(d) = score(d) × (1 + w × 2^(-ageInDays / 30)) +``` + +The weight defaults to **0** (disabled). The half-life is fixed at 30 days +(`DEFAULT_INTELLIGENT_SEARCH_RECENCY_HALF_LIFE_DAYS`); offline sweeps found the 30-90 day range +essentially flat, so exposing it would add a setting without adding reachable quality. + +Timestamps come from the pipeline fragment metadata, so the boost costs no extra database work. +Candidates without a usable timestamp keep their relevance score rather than being penalised. + +The boost is bounded by `1 + w`, at most doubling a candidate's RRF score. RRF compresses rank +differences, so a high recency weight can move a fresh message substantially up the candidate list. +It cannot overturn an RRF score gap greater than that multiplier. + +## Candidate pool + +Each branch is asked for more candidates than the caller requested (`limit × 3`, clamped to +`[20, 100]`). Fusion needs overlap to work with, and results are filtered for visibility and room +subscription **after** retrieval, so a pool the size of the page would return short pages. The cap stays +above `MAX_INTELLIGENT_SEARCH_RESULTS` for that reason. Each branch retains its highest-ranked fragment +per message. The complete fused union (at most 200 messages) reaches temporal reranking and visibility +filtering before the requested page is selected. Over-fetching reduces short pages but cannot guarantee +a full page if too few retrieved messages remain accessible. diff --git a/packages/ai-search/src/constants.ts b/packages/ai-search/src/constants.ts index 961aa578f2a49..c5a4000af50ba 100644 --- a/packages/ai-search/src/constants.ts +++ b/packages/ai-search/src/constants.ts @@ -5,6 +5,15 @@ export const AI_SEARCH_RESULTS_PAGE_SIZE = 8; export const AI_SEARCH_FILTER_SUGGESTION_LIMIT = 5; export const AI_SEARCH_ROOM_LOOKUP_LIMIT = 20; export const MAX_INTELLIGENT_SEARCH_RESULTS = 50; +// Per-retriever candidate pool. Internal, and sized on a measured quality/latency frontier rather than +// quality alone. The cap must stay above MAX_INTELLIGENT_SEARCH_RESULTS, otherwise the largest page +// over-fetches nothing and permission filtering can return a short page. +export const INTELLIGENT_SEARCH_CANDIDATE_MULTIPLIER = 3; +export const MIN_INTELLIGENT_SEARCH_CANDIDATES = 20; +export const MAX_INTELLIGENT_SEARCH_CANDIDATES = 100; +export const INTELLIGENT_SEARCH_RRF_CONSTANT = 60; +export const DEFAULT_INTELLIGENT_SEARCH_SEMANTIC_WEIGHT = 50; +export const DEFAULT_INTELLIGENT_SEARCH_RECENCY_HALF_LIFE_DAYS = 30; export const MAX_SEARCH_FILTER_VALUES = 25; export const MAX_ROOM_SEARCH_PATTERN_LENGTH = 64; export const MAX_AI_SERVICE_RESPONSE_SIZE = 5 * 1024 * 1024; diff --git a/packages/ai-search/src/fusion.spec.ts b/packages/ai-search/src/fusion.spec.ts new file mode 100644 index 0000000000000..0e0389f620162 --- /dev/null +++ b/packages/ai-search/src/fusion.spec.ts @@ -0,0 +1,229 @@ +import { INTELLIGENT_SEARCH_RRF_CONSTANT } from './constants'; +import { + applyTemporalRerank, + filterSemanticCandidatesByMinimumSimilarity, + fuseCandidatesWithWeightedRRF, + getRecencyDecay, + toRankedCandidates, +} from './fusion'; +import type { FusedIntelligentSearchCandidate, IntelligentSearchCandidate } from './types'; + +const candidate = (msgId: string, overrides: Partial = {}): IntelligentSearchCandidate => ({ + _id: msgId, + msgId, + rid: 'room', + pipelineText: `${msgId} text`, + ...overrides, +}); + +const ids = (candidates: Pick[]): (string | undefined)[] => candidates.map(({ msgId }) => msgId); + +describe('AI Search fusion helpers', () => { + describe('filterSemanticCandidatesByMinimumSimilarity', () => { + it('keeps every candidate when the guardrail is disabled', () => { + const candidates = [candidate('m1', { semanticSimilarity: 0.1 }), candidate('m2', { semanticSimilarity: 0.9 })]; + + expect(filterSemanticCandidatesByMinimumSimilarity(candidates, 0)).toEqual(candidates); + expect(filterSemanticCandidatesByMinimumSimilarity(candidates, Number.NaN)).toEqual(candidates); + }); + + it('drops candidates below the minimum similarity', () => { + const candidates = [ + candidate('m1', { semanticSimilarity: 0.82 }), + candidate('m2', { semanticSimilarity: 0.69 }), + candidate('m3', { semanticSimilarity: 0.7 }), + ]; + + expect(ids(filterSemanticCandidatesByMinimumSimilarity(candidates, 70))).toEqual(['m1', 'm3']); + }); + + it('preserves unscored semantic candidates if the pipeline omits similarity metadata', () => { + const candidates = [candidate('m1'), candidate('m2', { semanticSimilarity: 0.1 })]; + + expect(ids(filterSemanticCandidatesByMinimumSimilarity(candidates, 70))).toEqual(['m1']); + }); + }); + + describe('fuseCandidatesWithWeightedRRF', () => { + const semantic = [candidate('s1'), candidate('s2'), candidate('shared')]; + const keyword = [candidate('shared'), candidate('k1'), candidate('k2')]; + + it('fuses on rank only, so incompatible retriever score scales never meet', () => { + const highDistanceSemantic = [candidate('s1', { score: 0.02, semanticSimilarity: 0.02 })]; + const highScoreKeyword = [candidate('k1', { score: 0.99 })]; + + // the semantic branch wins purely because it outranks on its own list, not because 0.99 > 0.02 + expect(ids(fuseCandidatesWithWeightedRRF(highDistanceSemantic, highScoreKeyword, 90, 10))).toEqual(['s1', 'k1']); + }); + + it('rewards candidates returned by both retrievers', () => { + const fused = fuseCandidatesWithWeightedRRF(semantic, keyword, 50, 10); + + expect(fused[0].msgId).toBe('shared'); + expect(fused[0].semanticRank).toBe(3); + expect(fused[0].fulltextRank).toBe(1); + }); + + it('counts a message once per branch and assigns consecutive unique-message ranks', () => { + const fused = fuseCandidatesWithWeightedRRF( + [candidate('repeated'), candidate('repeated'), candidate('shared')], + [candidate('shared')], + 50, + 10, + ); + + expect(ids(fused)).toEqual(['shared', 'repeated']); + expect(fused[0].semanticRank).toBe(2); + expect(fused[1].rrfScore).toBeCloseTo(0.5 / (INTELLIGENT_SEARCH_RRF_CONSTANT + 1), 10); + }); + + it('excludes candidates from a branch with zero weight', () => { + expect(ids(fuseCandidatesWithWeightedRRF(semantic, keyword, 100, 10))).toEqual(['s1', 's2', 'shared']); + expect(ids(fuseCandidatesWithWeightedRRF(semantic, keyword, 0, 10))).toEqual(['shared', 'k1', 'k2']); + }); + + it('shifts the ordering as the balance moves towards semantic', () => { + expect(ids(fuseCandidatesWithWeightedRRF(semantic, keyword, 10, 3))).toEqual(['shared', 'k1', 'k2']); + // 'shared' still leads at 90: its keyword rank 1 tops up an otherwise last-place semantic rank 3 + expect(ids(fuseCandidatesWithWeightedRRF(semantic, keyword, 90, 3))).toEqual(['shared', 's1', 's2']); + }); + + it('applies the documented weighted RRF contribution', () => { + const [top] = fuseCandidatesWithWeightedRRF([candidate('only')], [], 60, 1); + const k = INTELLIGENT_SEARCH_RRF_CONSTANT; + + expect(top.rrfScore).toBeCloseTo(0.6 / (k + 1), 10); + }); + + it('degrades to the populated branch when the other retriever returns nothing', () => { + expect(ids(fuseCandidatesWithWeightedRRF(semantic, [], 50, 10))).toEqual(['s1', 's2', 'shared']); + expect(ids(fuseCandidatesWithWeightedRRF([], keyword, 50, 10))).toEqual(['shared', 'k1', 'k2']); + }); + + it('keeps the semantic similarity of a candidate found by both retrievers', () => { + const [top] = fuseCandidatesWithWeightedRRF( + [candidate('shared', { score: 0.83, semanticSimilarity: 0.83, semanticDistance: 0.17 })], + [candidate('shared')], + 50, + 1, + ); + + expect(top.score).toBe(0.83); + expect(top.semanticSimilarity).toBe(0.83); + }); + + it('clamps out-of-range weights and respects the limit', () => { + expect(ids(fuseCandidatesWithWeightedRRF(semantic, keyword, 500, 2))).toEqual(['s1', 's2']); + expect(ids(fuseCandidatesWithWeightedRRF(semantic, keyword, -20, 2))).toEqual(['shared', 'k1']); + }); + + it('does not fuse branch-local synthetic ids into a single candidate', () => { + const semanticOnly = { _id: 'intelligent-semantic-0', rid: 'r1', pipelineText: 'a' }; + const keywordOnly = { _id: 'intelligent-keyword-0', rid: 'r2', pipelineText: 'b' }; + + const fused = fuseCandidatesWithWeightedRRF([semanticOnly], [keywordOnly], 50, 10); + + expect(fused).toHaveLength(2); + expect(fused.every(({ rrfScore }) => rrfScore === 0.5 / (INTELLIGENT_SEARCH_RRF_CONSTANT + 1))).toBe(true); + }); + + it('ignores candidates without any usable identifier', () => { + const unidentified = { _id: '', msgId: '', rid: 'room', pipelineText: '' }; + + expect(fuseCandidatesWithWeightedRRF([unidentified], [], 50, 10)).toEqual([]); + }); + }); + + describe('toRankedCandidates', () => { + it('gives single-retriever results the same rank-based score shape as fusion', () => { + const ranked = toRankedCandidates([candidate('m1'), candidate('m2')]); + + expect(ranked[0].rrfScore).toBeCloseTo(1 / (INTELLIGENT_SEARCH_RRF_CONSTANT + 1), 10); + expect(ranked[0].semanticRank).toBe(1); + expect(ranked[1].semanticRank).toBe(2); + }); + + it('records the keyword rank for keyword-sourced candidates', () => { + const [ranked] = toRankedCandidates([candidate('m1', { source: 'keyword' })]); + + expect(ranked.fulltextRank).toBe(1); + expect(ranked.semanticRank).toBeUndefined(); + }); + }); + + describe('getRecencyDecay', () => { + it('halves the decay every half-life', () => { + expect(getRecencyDecay(0, 30)).toBe(1); + expect(getRecencyDecay(30, 30)).toBeCloseTo(0.5, 10); + expect(getRecencyDecay(60, 30)).toBeCloseTo(0.25, 10); + }); + + it('treats future timestamps as brand new and rejects an unusable half-life', () => { + expect(getRecencyDecay(-5, 30)).toBe(1); + expect(getRecencyDecay(10, 0)).toBe(0); + expect(getRecencyDecay(Number.NaN, 30)).toBe(0); + }); + }); + + describe('applyTemporalRerank', () => { + const now = new Date('2026-09-11T12:00:00.000Z'); + const daysAgo = (days: number): string => new Date(now.getTime() - days * 24 * 60 * 60 * 1000).toISOString(); + const ranked = (entries: [string, number, string][]): FusedIntelligentSearchCandidate[] => + entries.map(([msgId, rrfScore, ts]) => ({ ...candidate(msgId, { ts }), rrfScore })); + + it('is a no-op when the boost is disabled', () => { + const candidates = ranked([ + ['old', 0.02, daysAgo(400)], + ['new', 0.01, daysAgo(0)], + ]); + + expect(applyTemporalRerank(candidates, { recencyWeight: 0, halfLifeDays: 30, now })).toEqual(candidates); + }); + + it('promotes a fresher candidate over a slightly more relevant stale one', () => { + const candidates = ranked([ + ['stale', 0.016, daysAgo(365)], + ['fresh', 0.015, daysAgo(0)], + ]); + + expect(ids(applyTemporalRerank(candidates, { recencyWeight: 100, halfLifeDays: 30, now }))).toEqual(['fresh', 'stale']); + }); + + it('cannot overturn a large relevance gap', () => { + const candidates = ranked([ + ['relevant', 0.05, daysAgo(365)], + ['fresh', 0.015, daysAgo(0)], + ]); + + expect(ids(applyTemporalRerank(candidates, { recencyWeight: 100, halfLifeDays: 30, now }))).toEqual(['relevant', 'fresh']); + }); + + it('leaves candidates without a usable timestamp at their relevance score rather than penalising them', () => { + const candidates: FusedIntelligentSearchCandidate[] = [ + { ...candidate('no-ts'), rrfScore: 0.02 }, + { ...candidate('bad-ts', { ts: 'not-a-date' }), rrfScore: 0.019 }, + { ...candidate('fresh', { ts: daysAgo(0) }), rrfScore: 0.018 }, + ]; + + expect(ids(applyTemporalRerank(candidates, { recencyWeight: 50, halfLifeDays: 30, now }))).toEqual(['fresh', 'no-ts', 'bad-ts']); + }); + + it('keeps the incoming order for ties', () => { + const candidates = ranked([ + ['first', 0.02, daysAgo(10)], + ['second', 0.02, daysAgo(10)], + ]); + + expect(ids(applyTemporalRerank(candidates, { recencyWeight: 40, halfLifeDays: 30, now }))).toEqual(['first', 'second']); + }); + + it('ignores an unusable half-life instead of dropping the boost silently', () => { + const candidates = ranked([ + ['stale', 0.016, daysAgo(365)], + ['fresh', 0.015, daysAgo(0)], + ]); + + expect(ids(applyTemporalRerank(candidates, { recencyWeight: 100, halfLifeDays: 0, now }))).toEqual(['stale', 'fresh']); + }); + }); +}); diff --git a/packages/ai-search/src/fusion.ts b/packages/ai-search/src/fusion.ts new file mode 100644 index 0000000000000..18c4d93da6f66 --- /dev/null +++ b/packages/ai-search/src/fusion.ts @@ -0,0 +1,147 @@ +import { INTELLIGENT_SEARCH_RRF_CONSTANT } from './constants'; +import type { + FusedIntelligentSearchCandidate, + IntelligentSearchCandidate, + IntelligentSearchCandidateSource, + TemporalRerankOptions, +} from './types'; + +const clampPercent = (value: unknown): number => { + const numeric = Number(value); + if (!Number.isFinite(numeric)) { + return 0; + } + + return Math.min(100, Math.max(0, Math.floor(numeric))); +}; + +const getCandidateId = (candidate: IntelligentSearchCandidate): string => candidate.msgId || candidate._id; + +// Deliberately does not apply to keyword candidates: an exact error code or ticket id must not be +// dropped for being semantically unremarkable. +export const filterSemanticCandidatesByMinimumSimilarity = ( + candidates: IntelligentSearchCandidate[], + minimumSimilarityPercent: number, +): IntelligentSearchCandidate[] => { + const minimumSimilarity = clampPercent(minimumSimilarityPercent); + if (!minimumSimilarity) { + return candidates; + } + + const threshold = minimumSimilarity / 100; + return candidates.filter((candidate) => candidate.semanticSimilarity === undefined || candidate.semanticSimilarity >= threshold); +}; + +// Fuses on rank only: the retrievers report cosine distance and full-text rank respectively, so their +// raw scores are not comparable and must never meet. `semanticWeight` is the 0-100 admin balance. +export const fuseCandidatesWithWeightedRRF = ( + semanticCandidates: IntelligentSearchCandidate[], + keywordCandidates: IntelligentSearchCandidate[], + semanticWeight: number, + limit: number, + rrfConstant: number = INTELLIGENT_SEARCH_RRF_CONSTANT, +): FusedIntelligentSearchCandidate[] => { + const normalizedSemanticWeight = clampPercent(semanticWeight) / 100; + const normalizedKeywordWeight = 1 - normalizedSemanticWeight; + + const fused = new Map(); + const addBranch = (candidates: IntelligentSearchCandidate[], branch: IntelligentSearchCandidateSource, branchWeight: number): void => { + if (!branchWeight) { + return; + } + const seen = new Set(); + for (const candidate of candidates) { + const candidateId = getCandidateId(candidate); + if (!candidateId || seen.has(candidateId)) { + continue; + } + seen.add(candidateId); + + const rank = seen.size; + const contribution = branchWeight / (rrfConstant + rank); + const existing = fused.get(candidateId); + if (!existing) { + fused.set(candidateId, { + ...candidate, + rrfScore: contribution, + ...(branch === 'semantic' ? { semanticRank: rank } : { fulltextRank: rank }), + }); + continue; + } + + fused.set(candidateId, { + ...existing, + ts: existing.ts || candidate.ts, + rrfScore: existing.rrfScore + contribution, + ...(branch === 'semantic' ? { semanticRank: rank } : { fulltextRank: rank }), + }); + } + }; + + addBranch(semanticCandidates, 'semantic', normalizedSemanticWeight); + addBranch(keywordCandidates, 'keyword', normalizedKeywordWeight); + + return [...fused.values()] + .sort((a, b) => { + if (b.rrfScore !== a.rrfScore) { + return b.rrfScore - a.rrfScore; + } + + const semanticRankDelta = (a.semanticRank ?? Number.MAX_SAFE_INTEGER) - (b.semanticRank ?? Number.MAX_SAFE_INTEGER); + if (semanticRankDelta !== 0) { + return semanticRankDelta; + } + + return (a.fulltextRank ?? Number.MAX_SAFE_INTEGER) - (b.fulltextRank ?? Number.MAX_SAFE_INTEGER); + }) + .slice(0, limit); +}; + +// Gives single-retriever results the same rank-based score as fusion, so every mode reaches the +// temporal stage comparably scored. +export const toRankedCandidates = ( + candidates: IntelligentSearchCandidate[], + rrfConstant: number = INTELLIGENT_SEARCH_RRF_CONSTANT, +): FusedIntelligentSearchCandidate[] => + candidates.map((candidate, index) => ({ + ...candidate, + rrfScore: 1 / (rrfConstant + index + 1), + ...(candidate.source === 'keyword' ? { fulltextRank: index + 1 } : { semanticRank: index + 1 }), + })); + +export const getRecencyDecay = (ageInDays: number, halfLifeDays: number): number => { + if (!(halfLifeDays > 0) || !Number.isFinite(ageInDays)) { + return 0; + } + + return 2 ** (-Math.max(0, ageInDays) / halfLifeDays); +}; + +// The boost is bounded by `1 + recencyWeight`. Missing or unparseable timestamps keep the RRF score. +export const applyTemporalRerank = ( + candidates: FusedIntelligentSearchCandidate[], + { recencyWeight, halfLifeDays, now = new Date() }: TemporalRerankOptions, +): FusedIntelligentSearchCandidate[] => { + const normalizedRecencyWeight = clampPercent(recencyWeight) / 100; + if (!normalizedRecencyWeight || !(halfLifeDays > 0)) { + return candidates; + } + + const nowMs = now.getTime(); + const millisecondsPerDay = 24 * 60 * 60 * 1000; + + return candidates + .map((candidate, index) => { + const timestamp = candidate.ts ? new Date(candidate.ts).getTime() : Number.NaN; + if (Number.isNaN(timestamp)) { + return { candidate, index, score: candidate.rrfScore }; + } + + const ageInDays = (nowMs - timestamp) / millisecondsPerDay; + const decay = getRecencyDecay(ageInDays, halfLifeDays); + + return { candidate, index, score: candidate.rrfScore * (1 + normalizedRecencyWeight * decay) }; + }) + .sort((a, b) => (b.score !== a.score ? b.score - a.score : a.index - b.index)) + .map(({ candidate }) => candidate); +}; diff --git a/packages/ai-search/src/index.ts b/packages/ai-search/src/index.ts index e3f7bd0f029af..effd4e72f66a6 100644 --- a/packages/ai-search/src/index.ts +++ b/packages/ai-search/src/index.ts @@ -1,5 +1,6 @@ export * from './clientSearch'; export * from './constants'; +export * from './fusion'; export * from './intelligentSearch'; export * from './llm'; export type * from './types'; diff --git a/packages/ai-search/src/intelligentSearch.spec.ts b/packages/ai-search/src/intelligentSearch.spec.ts index 5b917c447a504..9315033cbb6a8 100644 --- a/packages/ai-search/src/intelligentSearch.spec.ts +++ b/packages/ai-search/src/intelligentSearch.spec.ts @@ -27,6 +27,23 @@ describe('AI Search intelligent search helpers', () => { }); describe('normalizeIntelligentSearchCandidates', () => { + it.each(['semantic', 'keyword'] as const)('counts unique messages toward the %s candidate limit', (source) => { + const results = normalizeIntelligentSearchCandidates( + [ + { id: 'm1', score: 0.1 }, + { id: 'm1', score: 0.2 }, + { id: 'm2', score: 0.3 }, + ], + [], + 2, + undefined, + source, + ); + + expect(results.map(({ msgId }) => msgId)).toEqual(['m1', 'm2']); + expect(results[0].semanticSimilarity).toBe(source === 'semantic' ? 0.9 : undefined); + }); + it('normalizes supported pipeline response shapes and score formats', () => { const results = normalizeIntelligentSearchCandidates( { @@ -43,10 +60,37 @@ describe('AI Search intelligent search helpers', () => { ); expect(results).toEqual([ - { _id: 'm1', rid: 'r1', msgId: 'm1', pipelineText: 'metadata text', score: 0.89 }, - { _id: 'm2', rid: 'r2', msgId: 'm2', pipelineText: 'content text', score: 0.49 }, - { _id: 'm3', rid: 'r3', msgId: 'm3', pipelineText: 'document text', score: 0.88 }, - { _id: 'm4', rid: 'r4', msgId: 'm4', pipelineText: 'no numeric score' }, + { + _id: 'm1', + rid: 'r1', + msgId: 'm1', + pipelineText: 'metadata text', + score: 0.89, + semanticSimilarity: 0.89, + semanticDistance: 0.11, + source: 'semantic', + }, + { + _id: 'm2', + rid: 'r2', + msgId: 'm2', + pipelineText: 'content text', + score: 0.49, + semanticSimilarity: 0.49, + semanticDistance: 0.51, + source: 'semantic', + }, + { + _id: 'm3', + rid: 'r3', + msgId: 'm3', + pipelineText: 'document text', + score: 0.88, + semanticSimilarity: 0.88, + semanticDistance: 0.12, + source: 'semantic', + }, + { _id: 'm4', rid: 'r4', msgId: 'm4', pipelineText: 'no numeric score', source: 'semantic' }, ]); }); @@ -58,7 +102,7 @@ describe('AI Search intelligent search helpers', () => { expect(normalizeIntelligentSearchCandidates(rawResults, [], 10)).toHaveLength(2); expect(normalizeIntelligentSearchCandidates(rawResults, ['allowed'], 10)).toEqual([ - { _id: 'm1', rid: 'allowed', msgId: 'm1', pipelineText: 'allowed' }, + { _id: 'm1', rid: 'allowed', msgId: 'm1', pipelineText: 'allowed', source: 'semantic' }, ]); }); @@ -69,7 +113,84 @@ describe('AI Search intelligent search helpers', () => { 1, ); - expect(results).toEqual([{ _id: 'm1', rid: 'r1', msgId: 'm1', pipelineText: '' }]); + expect(results).toEqual([{ _id: 'm1', rid: 'r1', msgId: 'm1', pipelineText: '', source: 'semantic' }]); + }); + + it('marks keyword candidates without adding a semantic score', () => { + expect( + normalizeIntelligentSearchCandidates( + { results: [{ metadata: { room_id: 'r1', msg_id: 'm1' }, text: 'keyword match', score: 0.42 }] }, + ['r1'], + 10, + undefined, + 'keyword', + ), + ).toEqual([ + { + _id: 'm1', + rid: 'r1', + msgId: 'm1', + pipelineText: 'keyword match', + source: 'keyword', + }, + ]); + }); + }); + + describe('keyword candidate scores', () => { + it('never reports a full-text rank as a semantic similarity', () => { + // 0.2803 is the stronger lexical hit; read as a distance it would display as the weaker one + const [best, worst] = normalizeIntelligentSearchCandidates( + { + results: [ + { metadata: { room_id: 'r1', msg_id: 'm1' }, score: 0.2803 }, + { metadata: { room_id: 'r2', msg_id: 'm2' }, score: 0.0183 }, + ], + }, + [], + 10, + undefined, + 'keyword', + ); + + expect(best).not.toHaveProperty('score'); + expect(best).not.toHaveProperty('semanticSimilarity'); + expect(best).not.toHaveProperty('semanticDistance'); + expect(worst).not.toHaveProperty('score'); + expect(best.source).toBe('keyword'); + }); + + it('still reports semantic similarity for semantic candidates', () => { + const [candidate] = normalizeIntelligentSearchCandidates( + { results: [{ metadata: { room_id: 'r1', msg_id: 'm1' }, score: 0.2 }] }, + [], + 10, + undefined, + 'semantic', + ); + + expect(candidate.score).toBe(0.8); + expect(candidate.semanticSimilarity).toBe(0.8); + }); + }); + + describe('candidate timestamps', () => { + it('carries the pipeline timestamp through for the temporal rerank stage', () => { + const [withMetadataTs, withResultTs, withoutTs] = normalizeIntelligentSearchCandidates( + { + results: [ + { metadata: { room_id: 'r1', msg_id: 'm1', timestamp: '2026-01-05T12:00:00.000Z' } }, + { metadata: { room_id: 'r2', msg_id: 'm2' }, timestamp: '2026-02-05T12:00:00.000Z' }, + { metadata: { room_id: 'r3', msg_id: 'm3' } }, + ], + }, + [], + 10, + ); + + expect(withMetadataTs.ts).toBe('2026-01-05T12:00:00.000Z'); + expect(withResultTs.ts).toBe('2026-02-05T12:00:00.000Z'); + expect(withoutTs).not.toHaveProperty('ts'); }); }); @@ -167,7 +288,50 @@ describe('AI Search intelligent search helpers', () => { }); }); - it('returns an empty result set for non-2xx pipeline responses', async () => { + it('requests keyword mode and omits threshold when semantic filtering is not applicable', async () => { + let requestBody = ''; + const fetch: AIServiceFetch = async (_url, options) => { + requestBody = String(options.body); + + return { + ok: true, + status: 200, + json: async () => ({ results: [] }), + text: async () => '', + }; + }; + + await searchIntelligentPipeline({ + query: 'service health', + config: { + baseUrl: 'https://pipeline.example.com/', + pipelineId: 'workspace', + apiKey: 'key', + apiKeySecret: 'secret', + minimumSimilarityPercent: 70, + }, + classifications: ['user'], + pipelineFilters: { room_id: { $in: ['r1'] } }, + limit: 5, + fetch, + mode: 'keyword', + }); + + expect(JSON.parse(requestBody)).toEqual({ + query: 'service health', + type: 'search', + classification: { + classifications: ['user'], + search_type: 1, + }, + filters: { room_id: { $in: ['r1'] } }, + params: { + k: 5, + }, + }); + }); + + it('rejects non-2xx responses so orchestration can distinguish failure from no matches', async () => { const fetch: AIServiceFetch = async () => ({ ok: false, status: 500, @@ -175,7 +339,7 @@ describe('AI Search intelligent search helpers', () => { text: async () => 'failed', }); - const result = await searchIntelligentPipeline({ + const result = searchIntelligentPipeline({ query: 'fruit colors', config: { baseUrl: 'https://pipeline.example.com', @@ -189,7 +353,7 @@ describe('AI Search intelligent search helpers', () => { fetch, }); - expect(result).toEqual([]); + await expect(result).rejects.toThrow('Intelligent search pipeline returned HTTP 500'); }); }); }); diff --git a/packages/ai-search/src/intelligentSearch.ts b/packages/ai-search/src/intelligentSearch.ts index 7016472936e5f..9a9cb79c6cb83 100644 --- a/packages/ai-search/src/intelligentSearch.ts +++ b/packages/ai-search/src/intelligentSearch.ts @@ -3,6 +3,7 @@ import type { AIServiceFetch, AIServiceLogger, IntelligentSearchCandidate, + IntelligentSearchCandidateSource, IntelligentSearchFilters, IntelligentSearchPipelineFilters, IntelligentSearchPipelineRequest, @@ -55,26 +56,47 @@ export const normalizeSimilarityPercent = (value: unknown): number => { export const getSemanticDistanceThreshold = (minimumSimilarityPercent: number): number => Number((1 - minimumSimilarityPercent / 100).toFixed(4)); -// pipeline contract: `score`/`distance` are cosine distances (lower is better, similarity = 1 - distance) -const normalizePipelineSimilarityScore = (value: number, type: 'distance' | 'similarity'): number => { +// Pipeline contract, verified against a live pipeline: `score`/`distance` are cosine distances (lower is +// better), `similarity` values are cosine similarities. Percentages are accepted for provider drift. +const normalizePipelineScore = (value: number): number => { const normalizedValue = Math.abs(value) > 1 ? value / 100 : value; - const similarity = type === 'distance' ? 1 - normalizedValue : normalizedValue; - return Math.min(1, Math.max(0, similarity)); + return Math.min(1, Math.max(0, normalizedValue)); }; -const extractPipelineSimilarityScore = (result: Record, metadata: Record): number | undefined => { +// The keyword retriever reuses `score` for a full-text rank where higher is better, so reading it as a +// distance would invert it and fabricate a confident similarity. +const extractPipelineSimilarityScores = ( + result: Record, + metadata: Record, + source: IntelligentSearchCandidateSource, +): { semanticSimilarity?: number; semanticDistance?: number } => { + if (source === 'keyword') { + return {}; + } + const similarity = firstNumber(result.similarity, metadata.similarity); if (typeof similarity === 'number') { - return normalizePipelineSimilarityScore(similarity, 'similarity'); + const semanticSimilarity = normalizePipelineScore(similarity); + const semanticDistance = Number((1 - semanticSimilarity).toFixed(4)); + + return { + semanticSimilarity, + semanticDistance, + }; } const distance = firstNumber(result.score, result.distance, metadata.score, metadata.distance); if (typeof distance === 'number') { - return normalizePipelineSimilarityScore(distance, 'distance'); + const semanticDistance = normalizePipelineScore(distance); + + return { + semanticSimilarity: Number((1 - semanticDistance).toFixed(4)), + semanticDistance, + }; } - return undefined; + return {}; }; const extractIntelligentResultIds = (result: Record): { rid?: string; msgId?: string } => { @@ -101,6 +123,7 @@ export const normalizeIntelligentSearchCandidates = ( userRoomIds: string[] = [], limit: number, logger?: AIServiceLogger, + source: IntelligentSearchCandidateSource = 'semantic', ): IntelligentSearchCandidate[] => { let rawResults: unknown[] = []; const rawSearchResultsRecord = asRecord(rawSearchResults); @@ -129,6 +152,7 @@ export const normalizeIntelligentSearchCandidates = ( const shouldFilterByRoomIds = userRoomIdSet.size > 0; const candidates: IntelligentSearchCandidate[] = []; + const seenMessageIds = new Set(); for (let index = 0; index < rawResults.length && candidates.length < limit; index++) { const result = asRecord(rawResults[index]); const metadata = asRecord(result.metadata); @@ -140,14 +164,29 @@ export const normalizeIntelligentSearchCandidates = ( logger?.debug?.({ msg: 'Intelligent search result filtered: room not in user subscriptions', rid }); continue; } + // A message can have several indexed fragments. Keep its highest-ranked fragment. + if (msgId) { + if (seenMessageIds.has(msgId)) { + continue; + } + seenMessageIds.add(msgId); + } - const score = extractPipelineSimilarityScore(result, metadata); + const { semanticDistance, semanticSimilarity } = extractPipelineSimilarityScores(result, metadata, source); + const ts = firstString(metadata.timestamp, result.timestamp); candidates.push({ - _id: msgId || `intelligent-${index}`, + // source-qualified: the index is per-retriever, so a bare index would fuse unrelated candidates + _id: msgId || `intelligent-${source}-${index}`, rid, msgId, pipelineText: firstString(result.text, result.content, result.document, result.page_content, metadata.text) || '', - ...(typeof score === 'number' && { score }), + ...(ts && { ts }), + ...(typeof semanticSimilarity === 'number' && { + score: semanticSimilarity, + semanticSimilarity, + semanticDistance, + }), + ...(source && { source }), }); } @@ -228,19 +267,24 @@ export const searchIntelligentPipeline = async ({ limit, fetch, logger, + mode = 'semantic', }: IntelligentSearchPipelineRequest): Promise => { const minimumSimilarity = normalizeSimilarityPercent(config.minimumSimilarityPercent); const formattedQuery = config.queryTemplate ? config.queryTemplate.replace('{query}', query) : query; const url = buildEndpointUrl(config.baseUrl, `pipelines/${encodeURIComponent(config.pipelineId)}/search`); + const searchType = mode === 'keyword' ? 'search' : 'similarity'; + const shouldApplyThreshold = mode !== 'keyword'; + const threshold = shouldApplyThreshold ? getSemanticDistanceThreshold(minimumSimilarity) : undefined; logger?.debug?.({ msg: 'Intelligent search request', url, queryLength: formattedQuery.length, hasQueryTemplate: Boolean(config.queryTemplate), + searchType, filterKeys: Object.keys(pipelineFilters), classificationCount: classifications.length, - threshold: getSemanticDistanceThreshold(minimumSimilarity), + threshold, }); let response: Awaited>; @@ -257,15 +301,15 @@ export const searchIntelligentPipeline = async ({ }, body: JSON.stringify({ query: formattedQuery, - type: 'similarity', + type: searchType, classification: { classifications, - search_type: 2, + search_type: mode === 'keyword' ? 1 : 2, }, filters: pipelineFilters, params: { k: limit, - threshold: getSemanticDistanceThreshold(minimumSimilarity), + ...(typeof threshold === 'number' && { threshold }), }, }), }); @@ -277,7 +321,7 @@ export const searchIntelligentPipeline = async ({ if (!response.ok) { const body = await response.text().catch(() => ''); logger?.warn?.({ msg: 'Intelligent search pipeline returned error', url, status: response.status, bodyLength: body.length }); - return []; + throw new Error(`Intelligent search pipeline returned HTTP ${response.status}`); } const json = await response.json(); diff --git a/packages/ai-search/src/types.ts b/packages/ai-search/src/types.ts index f98dd1245ba7e..97694bd662d61 100644 --- a/packages/ai-search/src/types.ts +++ b/packages/ai-search/src/types.ts @@ -50,6 +50,36 @@ export type IntelligentSearchPipelineConfig = { minimumSimilarityPercent?: number; }; +export type IntelligentSearchType = 'semantic' | 'keyword' | 'hybrid'; + +export type IntelligentSearchCandidateSource = 'semantic' | 'keyword'; + +export type IntelligentSearchCandidate = { + _id: string; + rid?: string; + msgId?: string; + pipelineText: string; + /** Normalized cosine similarity. Unset for keyword candidates, which have no comparable score. */ + score?: number; + semanticSimilarity?: number; + semanticDistance?: number; + source?: IntelligentSearchCandidateSource; + ts?: string; +}; + +export type FusedIntelligentSearchCandidate = IntelligentSearchCandidate & { + rrfScore: number; + semanticRank?: number; + fulltextRank?: number; +}; + +export type TemporalRerankOptions = { + /** 0 disables the boost, 100 doubles the freshest candidate's score */ + recencyWeight: number; + halfLifeDays: number; + now?: Date; +}; + export type IntelligentSearchFilters = { rid?: string; rids?: string[]; @@ -62,14 +92,6 @@ export type IntelligentSearchFilters = { export type IntelligentSearchPipelineFilters = Record; -export type IntelligentSearchCandidate = { - _id: string; - rid?: string; - msgId?: string; - pipelineText: string; - score?: number; -}; - export type IntelligentSearchPipelineRequest = { query: string; config: IntelligentSearchPipelineConfig; @@ -78,4 +100,6 @@ export type IntelligentSearchPipelineRequest = { limit: number; fetch: AIServiceFetch; logger?: AIServiceLogger; + /** a single request targets one retriever; `hybrid` is resolved before reaching this layer */ + mode?: IntelligentSearchCandidateSource; }; diff --git a/packages/core-services/src/types/IAISearchService.ts b/packages/core-services/src/types/IAISearchService.ts index c670d5e8d0a93..76bfddc9ab9f1 100644 --- a/packages/core-services/src/types/IAISearchService.ts +++ b/packages/core-services/src/types/IAISearchService.ts @@ -19,6 +19,8 @@ export type AISearchStatus = { answerGenerationConfigured: boolean; }; +export type AISearchType = 'semantic' | 'keyword' | 'hybrid'; + export type AISearchAnswerMessage = { text: string; username?: string; @@ -54,7 +56,13 @@ export type AISearchResult = { export interface IAISearchService extends IServiceClass { status(): Promise; - search(params: { query: string; userId: string; filters?: AISearchFilters; limit?: number }): Promise; + search(params: { + query: string; + userId: string; + filters?: AISearchFilters; + limit?: number; + searchType?: AISearchType; + }): Promise; answer(params: { query: string; messages: AISearchAnswerMessage[] }): Promise; diff --git a/packages/i18n/src/locales/en.i18n.json b/packages/i18n/src/locales/en.i18n.json index b3fb8c78dc088..4875a69d49423 100644 --- a/packages/i18n/src/locales/en.i18n.json +++ b/packages/i18n/src/locales/en.i18n.json @@ -588,6 +588,10 @@ "AI_Intelligent_Search_Pipeline_ID_Description": "Identifier of the target pipeline used for semantic search requests.", "AI_Intelligent_Search_Query_Template": "Query template", "AI_Intelligent_Search_Query_Template_Description": "Optional template applied to search queries before sending to the pipeline. Use {query} as the placeholder. Leave blank to send the raw query.", + "AI_Intelligent_Search_Recency_Weight": "Recency boost", + "AI_Intelligent_Search_Recency_Weight_Description": "How strongly newer messages are promoted after relevance ranking. A message keeps half of its boost every 30 days. Use 0 to rank purely by relevance.", + "AI_Intelligent_Search_Semantic_Weight": "Search balance", + "AI_Intelligent_Search_Semantic_Weight_Description": "Balance between keyword and semantic retrieval: 0 searches by keyword only, 100 by meaning only, and anything in between combines both. This is not the percentage of results that come from semantic search.", "AI_LLM_OpenAI_API_Key": "API key", "AI_LLM_OpenAI_API_Key_Description": "API key for the OpenAI-compatible chat completions endpoint.", "AI_LLM_OpenAI_Base_URL": "API base URL", diff --git a/packages/rest-typings/src/v1/aiSearch.ts b/packages/rest-typings/src/v1/aiSearch.ts index 8d36c9eeb4527..bdfb46af16b66 100644 --- a/packages/rest-typings/src/v1/aiSearch.ts +++ b/packages/rest-typings/src/v1/aiSearch.ts @@ -12,6 +12,7 @@ type AISearch = { fromUsernames?: string; startDate?: string; endDate?: string; + searchType?: 'semantic' | 'keyword' | 'hybrid'; }; const AISearchSchema = { @@ -38,6 +39,7 @@ const AISearchSchema = { { type: 'string', format: 'date-time' }, ], }, + searchType: { type: 'string', enum: ['semantic', 'keyword', 'hybrid'] }, }, required: ['query'], additionalProperties: false,