From 8dbd0f0802152baa197d5f31c04db7be5f751f0d Mon Sep 17 00:00:00 2001 From: Dnouv Date: Fri, 11 Sep 2026 20:03:45 +0800 Subject: [PATCH 01/10] feat: hybrid retrieval and temporal reranking for AI Search Adds keyword and hybrid retrieval modes alongside the existing semantic search. In hybrid mode both retrievers run in parallel and are fused with weighted Reciprocal Rank Fusion, balanced by a new 0-100 admin setting. Fusion works on rank positions only. The pipeline reports cosine *distance* for semantic hits (lower is better) and a full-text rank for keyword hits (higher is better), so the raw scores are never comparable and are never compared. The pipeline's own `type: "hybrid"` placeholder returns 501 and exposes no weight parameter, so fusion has to happen here regardless. The minimum semantic similarity guardrail now applies only to semantic candidates. An exact match on an error code or ticket id must not be discarded for being semantically unremarkable, which is exactly what hybrid search is for. An optional recency boost reranks after relevance using exponential half-life decay, reading timestamps from pipeline metadata so it costs no extra database work. It is disabled by default and leaves ranking unchanged until enabled. Also fixes two truncation bugs on the way through: fusion sliced to the page size before permission filtering ran, and normalizeIntelligentResults sliced again, so a hybrid search could return a short page whenever any candidate resolved to a non-visible message. Each retriever is now asked for a candidate pool instead of a page. Defaults are backed by an offline benchmark over a 547-document judged corpus; see docs/features/ai-search-hybrid-benchmark.md. --- .changeset/hybrid-ai-search-retrieval.md | 20 ++ apps/meteor/server/api/v1/ai-search.ts | 9 + .../server/services/ai-search/service.ts | 157 ++++++++++- apps/meteor/server/settings/ai.ts | 53 ++++ .../services/ai-search/service.tests.ts | 247 +++++++++++++++++- docs/features/ai-search-hybrid-benchmark.md | 101 +++++++ docs/features/ai-search-hybrid.md | 120 +++++++++ packages/ai-search/src/constants.ts | 9 + packages/ai-search/src/fusion.spec.ts | 201 ++++++++++++++ packages/ai-search/src/fusion.ts | 163 ++++++++++++ packages/ai-search/src/index.ts | 1 + .../ai-search/src/intelligentSearch.spec.ts | 125 ++++++++- packages/ai-search/src/intelligentSearch.ts | 58 +++- packages/ai-search/src/types.ts | 39 ++- .../src/types/IAISearchService.ts | 10 +- packages/i18n/src/locales/en.i18n.json | 11 + packages/rest-typings/src/v1/aiSearch.ts | 2 + 17 files changed, 1283 insertions(+), 43 deletions(-) create mode 100644 .changeset/hybrid-ai-search-retrieval.md create mode 100644 docs/features/ai-search-hybrid-benchmark.md create mode 100644 docs/features/ai-search-hybrid.md create mode 100644 packages/ai-search/src/fusion.spec.ts create mode 100644 packages/ai-search/src/fusion.ts diff --git a/.changeset/hybrid-ai-search-retrieval.md b/.changeset/hybrid-ai-search-retrieval.md new file mode 100644 index 0000000000000..9f06f1120c8c5 --- /dev/null +++ b/.changeset/hybrid-ai-search-retrieval.md @@ -0,0 +1,20 @@ +--- +'@rocket.chat/ai-search': minor +'@rocket.chat/core-services': minor +'@rocket.chat/rest-typings': minor +'@rocket.chat/i18n': minor +'@rocket.chat/meteor': minor +--- + +Adds hybrid retrieval and optional temporal reranking to AI Search. + +A new **Search method** setting selects semantic, keyword, or hybrid retrieval. In hybrid mode the +semantic and full-text retrievers run in parallel and are fused with weighted Reciprocal Rank Fusion, +balanced by a **Hybrid search balance** setting (0 is keyword only, 100 is semantic only). Fusion works +on rank positions, so the retrievers' incompatible score scales are never compared directly. + +The minimum semantic similarity guardrail now applies only to semantic candidates, so an exact match on +an error code or ticket id is no longer discarded for being semantically unremarkable. + +An optional **Recency boost** reranks results by age after relevance ranking, using an exponential +half-life decay. It is disabled by default and leaves ranking unchanged until an admin opts in. diff --git a/apps/meteor/server/api/v1/ai-search.ts b/apps/meteor/server/api/v1/ai-search.ts index 39290d189affd..58e50517c01af 100644 --- a/apps/meteor/server/api/v1/ai-search.ts +++ b/apps/meteor/server/api/v1/ai-search.ts @@ -132,6 +132,13 @@ const parseCommaList = (value: string | undefined): string[] => { }; const parseQueryDate = (value: string | undefined): Date | undefined => (value ? new Date(value) : undefined); +const parseSearchType = (value: string | undefined): 'semantic' | 'keyword' | 'hybrid' | undefined => { + if (value === 'semantic' || value === 'keyword' || value === 'hybrid') { + return value; + } + + return undefined; +}; const getRoomMap = async (roomIds: string[]): Promise>> => { if (!roomIds.length) { @@ -254,6 +261,7 @@ API.v1.get( const fromUsernames = parseCommaList(this.queryParams.fromUsernames); const startDate = parseQueryDate(this.queryParams.startDate); const endDate = parseQueryDate(this.queryParams.endDate); + const searchType = parseSearchType(this.queryParams.searchType); const aiSearchStatus = await AISearch.status().catch((error) => { this.logger.warn({ msg: 'AI search status unavailable', err: error }); @@ -283,6 +291,7 @@ API.v1.get( startDate: startDate?.toISOString(), endDate: endDate?.toISOString(), }, + searchType, limit: intelligentLimit, }); } catch (error) { diff --git a/apps/meteor/server/services/ai-search/service.ts b/apps/meteor/server/services/ai-search/service.ts index 67cb473dd26c4..f5ea6a687a2c3 100644 --- a/apps/meteor/server/services/ai-search/service.ts +++ b/apps/meteor/server/services/ai-search/service.ts @@ -1,14 +1,27 @@ import { AI_LICENSE_MODULE, AI_SEARCH_PAGE_SIZE, + applyTemporalRerank, buildIntelligentSearchPipelineFilters, + DEFAULT_INTELLIGENT_SEARCH_RECENCY_HALF_LIFE_DAYS, + DEFAULT_INTELLIGENT_SEARCH_SEMANTIC_WEIGHT, + filterSemanticCandidatesByMinimumSimilarity, + fuseCandidatesWithWeightedRRF, generateOpenAICompatibleSearchAnswer, + INTELLIGENT_SEARCH_CANDIDATE_MULTIPLIER, listOpenAICompatibleModels, + MAX_INTELLIGENT_SEARCH_CANDIDATES, MAX_SEARCH_ANSWER_MESSAGES, MAX_SEARCH_ANSWER_TEXT_LENGTH, MAX_SEARCH_FILTER_VALUES, + MIN_INTELLIGENT_SEARCH_CANDIDATES, normalizeIntelligentSearchCandidates, + toRankedCandidates, + type FusedIntelligentSearchCandidate, + type IntelligentSearchCandidate, + type IntelligentSearchType, searchIntelligentPipeline, + type IntelligentSearchPipelineFilters, type IntelligentSearchFilters, type IntelligentSearchPipelineConfig, type OpenAICompatibleProviderConfig, @@ -154,6 +167,123 @@ export class AISearchService extends ServiceClass implements IAISearchService { }; } + private getSearchMode(): IntelligentSearchType { + const configuredMode = settings.get('AI_Intelligent_Search_Mode'); + if (configuredMode === 'hybrid' || configuredMode === 'keyword' || configuredMode === 'semantic') { + return configuredMode; + } + + return 'semantic'; + } + + private normalizeSearchType(searchType: IntelligentSearchType | undefined): IntelligentSearchType { + if (searchType === 'hybrid' || searchType === 'keyword' || searchType === 'semantic') { + return searchType; + } + + return this.getSearchMode(); + } + + private getHybridWeight(): number { + const configuredWeight = Number(settings.get('AI_Intelligent_Search_Semantic_Weight')); + if (!Number.isFinite(configuredWeight)) { + return DEFAULT_INTELLIGENT_SEARCH_SEMANTIC_WEIGHT; + } + + return Math.min(100, Math.max(0, Math.floor(configuredWeight))); + } + + private getRecencyWeight(): number { + const configuredWeight = Number(settings.get('AI_Intelligent_Search_Recency_Weight')); + if (!Number.isFinite(configuredWeight)) { + return 0; + } + + return Math.min(100, Math.max(0, Math.floor(configuredWeight))); + } + + private getRecencyHalfLifeDays(): number { + const configuredHalfLife = Number(settings.get('AI_Intelligent_Search_Recency_Half_Life_Days')); + if (!Number.isFinite(configuredHalfLife) || configuredHalfLife <= 0) { + return DEFAULT_INTELLIGENT_SEARCH_RECENCY_HALF_LIFE_DAYS; + } + + return Math.floor(configuredHalfLife); + } + + private async queryPipelineCandidates({ + query, + config, + classifications, + pipelineFilters, + limit, + sourceMode, + }: { + query: string; + config: IntelligentSearchPipelineConfig; + classifications: string[]; + pipelineFilters: IntelligentSearchPipelineFilters; + limit: number; + sourceMode: 'semantic' | 'keyword'; + }): Promise { + const raw = await searchIntelligentPipeline({ + query, + config, + classifications, + pipelineFilters, + limit, + fetch: fetchWithSsrfValidation, + logger, + mode: sourceMode, + }); + + return normalizeIntelligentSearchCandidates(raw, [], limit, logger, sourceMode); + } + + private async buildSearchCandidatesForMode( + query: string, + config: IntelligentSearchPipelineConfig, + classifications: string[], + pipelineFilters: IntelligentSearchPipelineFilters, + limit: number, + searchMode: IntelligentSearchType, + ): Promise { + const candidateLimit = this.getSearchCandidateLimit(limit); + const queryBranch = (sourceMode: 'semantic' | 'keyword') => + this.queryPipelineCandidates({ query, config, classifications, pipelineFilters, limit: candidateLimit, sourceMode }); + + const minimumSimilarityPercent = Number(config.minimumSimilarityPercent || 0); + const semanticWeight = searchMode === 'hybrid' ? this.getHybridWeight() : undefined; + + // a hybrid search collapses to a single retriever at the extremes of the balance slider + if (searchMode === 'keyword' || semanticWeight === 0) { + return toRankedCandidates(await queryBranch('keyword')); + } + + if (searchMode === 'semantic' || semanticWeight === 100) { + return toRankedCandidates(filterSemanticCandidatesByMinimumSimilarity(await queryBranch('semantic'), minimumSimilarityPercent)); + } + + const [semanticCandidates, keywordCandidates] = await Promise.all([queryBranch('semantic'), queryBranch('keyword')]); + + return fuseCandidatesWithWeightedRRF( + filterSemanticCandidatesByMinimumSimilarity(semanticCandidates, minimumSimilarityPercent), + keywordCandidates, + semanticWeight ?? DEFAULT_INTELLIGENT_SEARCH_SEMANTIC_WEIGHT, + candidateLimit, + ); + } + + /** + * Retrieval depth per branch. Deliberately larger than the requested page so that fusion has something + * to fuse and so that permission filtering does not eat into the page. Not admin configurable. + */ + private getSearchCandidateLimit(requestedLimit: number): number { + const scaledLimit = Math.max(requestedLimit, AI_SEARCH_PAGE_SIZE) * INTELLIGENT_SEARCH_CANDIDATE_MULTIPLIER; + + return Math.min(MAX_INTELLIGENT_SEARCH_CANDIDATES, Math.max(MIN_INTELLIGENT_SEARCH_CANDIDATES, scaledLimit)); + } + async status(): Promise { const hasIntelligentSearchLicense = await License.hasModule(AI_LICENSE_MODULE); const intelligentSearchEnabled = settings.get('AI_Intelligent_Search_Enabled'); @@ -217,13 +347,14 @@ export class AISearchService extends ServiceClass implements IAISearchService { } private async normalizeIntelligentResults( - rawSearchResults: unknown, + searchCandidates: IntelligentSearchCandidate[], userId: string, limit = AI_SEARCH_PAGE_SIZE, ): Promise { - const candidates = normalizeIntelligentSearchCandidates(rawSearchResults, [], limit, logger); + // the whole candidate pool is resolved, not just the first page: permission filtering below can + // drop any candidate, and pre-slicing here would silently return a short page const msgIdSet = new Set(); - for (const { msgId } of candidates) { + for (const { msgId } of searchCandidates) { if (msgId) { msgIdSet.add(msgId); } @@ -249,7 +380,7 @@ export class AISearchService extends ServiceClass implements IAISearchService { ]); const normalizedResults: AISearchResult[] = []; - for (const result of candidates) { + for (const result of searchCandidates) { // candidates without a visible database message could surface stale pipeline text const dbMessage = result.msgId ? messageMap.get(result.msgId) : undefined; if (!dbMessage) { @@ -315,11 +446,13 @@ export class AISearchService extends ServiceClass implements IAISearchService { userId, filters: rawFilters, limit = AI_SEARCH_PAGE_SIZE, + searchType, }: { query: string; userId: string; filters?: AISearchFilters; limit?: number; + searchType?: IntelligentSearchType; }): Promise { const hasIntelligentSearchLicense = await License.hasModule(AI_LICENSE_MODULE); const intelligentSearchEnabled = settings.get('AI_Intelligent_Search_Enabled'); @@ -365,17 +498,15 @@ export class AISearchService extends ServiceClass implements IAISearchService { return []; } - const json = await searchIntelligentPipeline({ - query, - config, - classifications, - pipelineFilters, - limit, - fetch: fetchWithSsrfValidation, - logger, + const requestedMode = this.normalizeSearchType(searchType); + const candidates = await this.buildSearchCandidatesForMode(query, config, classifications, pipelineFilters, limit, requestedMode); + // relevance first, freshness second: the temporal boost only reorders what fusion already selected + const rerankedCandidates = applyTemporalRerank(candidates, { + recencyWeight: this.getRecencyWeight(), + halfLifeDays: this.getRecencyHalfLifeDays(), }); - return this.normalizeIntelligentResults(json, userId, limit); + return this.normalizeIntelligentResults(rerankedCandidates, userId, limit); } async answer({ query, messages }: { query: string; messages: AISearchAnswerMessage[] }): Promise { diff --git a/apps/meteor/server/settings/ai.ts b/apps/meteor/server/settings/ai.ts index 90470bcdf30fb..f21fbdef64345 100644 --- a/apps/meteor/server/settings/ai.ts +++ b/apps/meteor/server/settings/ai.ts @@ -52,6 +52,59 @@ export const createAISettings = async (): Promise => { i18nDescription: 'AI_Intelligent_Search_Enabled_Description', }); + await settingsRegistry.add('AI_Intelligent_Search_Mode', 'semantic', { + group: AI_SETTINGS_GROUP, + section: 'Intelligent_Search', + type: 'select', + values: [ + { key: 'semantic', i18nLabel: 'AI_Intelligent_Search_Mode_Semantic' }, + { key: 'keyword', i18nLabel: 'AI_Intelligent_Search_Mode_Keyword' }, + { key: 'hybrid', i18nLabel: 'AI_Intelligent_Search_Mode_Hybrid' }, + ], + i18nLabel: 'AI_Intelligent_Search_Mode', + i18nDescription: 'AI_Intelligent_Search_Mode_Description', + enterprise: true, + modules: [AI_LICENSE_MODULE], + invalidValue: 'semantic', + enableQuery: { _id: 'AI_Intelligent_Search_Enabled', value: true }, + }); + + await settingsRegistry.add('AI_Intelligent_Search_Semantic_Weight', 50, { + group: AI_SETTINGS_GROUP, + section: 'Intelligent_Search', + type: 'int', + i18nLabel: 'AI_Intelligent_Search_Semantic_Weight', + i18nDescription: 'AI_Intelligent_Search_Semantic_Weight_Description', + enterprise: true, + modules: [AI_LICENSE_MODULE], + invalidValue: 50, + enableQuery: { _id: 'AI_Intelligent_Search_Mode', value: 'hybrid' }, + }); + + await settingsRegistry.add('AI_Intelligent_Search_Recency_Weight', 0, { + group: AI_SETTINGS_GROUP, + section: 'Intelligent_Search', + type: 'int', + i18nLabel: 'AI_Intelligent_Search_Recency_Weight', + i18nDescription: 'AI_Intelligent_Search_Recency_Weight_Description', + enterprise: true, + modules: [AI_LICENSE_MODULE], + invalidValue: 0, + enableQuery: { _id: 'AI_Intelligent_Search_Enabled', value: true }, + }); + + await settingsRegistry.add('AI_Intelligent_Search_Recency_Half_Life_Days', 30, { + group: AI_SETTINGS_GROUP, + section: 'Intelligent_Search', + type: 'int', + i18nLabel: 'AI_Intelligent_Search_Recency_Half_Life_Days', + i18nDescription: 'AI_Intelligent_Search_Recency_Half_Life_Days_Description', + enterprise: true, + modules: [AI_LICENSE_MODULE], + invalidValue: 30, + enableQuery: { _id: 'AI_Intelligent_Search_Recency_Weight', value: { $gt: 0 } }, + }); + await settingsRegistry.add('AI_Intelligent_Search_Pipeline_Base_URL', '', { group: AI_SETTINGS_GROUP, section: 'Intelligent_Search', diff --git a/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts b/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts index b2ee75ce9fc9d..69182c378690f 100644 --- a/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts +++ b/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts @@ -71,6 +71,10 @@ const cursor = (items: T[]): CursorResult => ({ const settings: Record = { AI_Intelligent_Search_Enabled: true, + AI_Intelligent_Search_Mode: 'semantic', + AI_Intelligent_Search_Semantic_Weight: 50, + AI_Intelligent_Search_Recency_Weight: 0, + AI_Intelligent_Search_Recency_Half_Life_Days: 30, AI_Intelligent_Search_Pipeline_Base_URL: 'https://pipeline.example.com', AI_Intelligent_Search_Pipeline_ID: 'workspace', AI_Intelligent_Search_API_Key: 'key', @@ -115,6 +119,7 @@ describe('AISearchService', () => { Subscriptions.findByUserIdAndRoomIds.callsFake((_userId: string, roomIds: string[]) => cursor(roomIds.filter((roomId) => roomId === 'allowed' || roomId === 'room-general').map((rid) => ({ rid }))), ); + Subscriptions.findByUserId.callsFake(() => cursor([{ rid: 'allowed' }])); Messages.findVisibleByIds.callsFake((msgIds: string[]) => cursor( msgIds.map((msgId) => ({ @@ -196,7 +201,8 @@ describe('AISearchService', () => { const [, options] = serverFetch.firstCall.args; const body = JSON.parse(options.body); - expect(body.params.k).to.equal(5); + // the retriever is asked for a candidate pool, not the requested page + expect(body.params.k).to.equal(50); expect(body.filters).to.deep.equal({ room_id: { $in: subscribedRoomIds }, }); @@ -216,6 +222,245 @@ describe('AISearchService', () => { ).to.be.true; }); + it('uses explicit searchType keyword when requested', async () => { + serverFetch.resolves({ + ok: true, + status: 200, + json: async () => ({ + results: [{ metadata: { room_id: 'allowed', msg_id: 'allowed-msg' }, text: 'keyword pipeline text', score: 0.77 }], + }), + text: async () => '', + }); + + await createService().search({ + query: 'fruit', + userId: 'user-id', + limit: 5, + searchType: 'keyword', + }); + + const requestBody = JSON.parse(serverFetch.firstCall.args[1].body); + expect(requestBody.type).to.equal('search'); + expect(requestBody.classification).to.deep.equal({ classifications: ['user', 'admin'], search_type: 1 }); + expect(requestBody.params).to.not.have.property('threshold'); + }); + + it('falls back to keyword path when hybrid semantic weight is 0', async () => { + cachedSettings.get.callsFake((key: string) => { + if (key === 'AI_Intelligent_Search_Semantic_Weight') { + return 0; + } + if (key === 'AI_Intelligent_Search_Mode') { + return 'hybrid'; + } + + return settings[key]; + }); + serverFetch.resolves({ + ok: true, + status: 200, + json: async () => ({ + results: [{ metadata: { room_id: 'allowed', msg_id: 'keyword-msg' }, text: 'keyword text', score: 0.9 }], + }), + text: async () => '', + }); + + await createService().search({ query: 'fruit', userId: 'user-id', searchType: 'hybrid' }); + + expect(serverFetch.callCount).to.equal(1); + const requestBody = JSON.parse(serverFetch.firstCall.args[1].body); + expect(requestBody.type).to.equal('search'); + }); + + it('falls back to semantic path when hybrid semantic weight is 100', async () => { + cachedSettings.get.callsFake((key: string) => { + if (key === 'AI_Intelligent_Search_Semantic_Weight') { + return 100; + } + if (key === 'AI_Intelligent_Search_Mode') { + return 'hybrid'; + } + + return settings[key]; + }); + serverFetch.resolves({ + ok: true, + status: 200, + json: async () => ({ + results: [{ metadata: { room_id: 'allowed', msg_id: 'semantic-msg' }, text: 'semantic text', score: 0.88 }], + }), + text: async () => '', + }); + + await createService().search({ query: 'fruit', userId: 'user-id', searchType: 'hybrid' }); + + expect(serverFetch.callCount).to.equal(1); + const requestBody = JSON.parse(serverFetch.firstCall.args[1].body); + expect(requestBody.type).to.equal('similarity'); + expect(requestBody.params).to.have.property('threshold'); + }); + + it('uses weighted hybrid with semantic threshold filtering only on semantic branch', async () => { + cachedSettings.get.callsFake((key: string) => + key === 'AI_Intelligent_Search_Mode' || key === 'AI_Intelligent_Search_Semantic_Weight' ? settings[key] : settings[key], + ); + + serverFetch.reset(); + serverFetch + .onCall(0) + .resolves({ + ok: true, + status: 200, + json: async () => ({ + results: [ + { metadata: { room_id: 'allowed', msg_id: 'allowed-msg' }, text: 'semantic good', score: 0.2 }, + { metadata: { room_id: 'allowed', msg_id: 'filtered-msg' }, text: 'semantic filtered', score: 0.49 }, + ], + }), + text: async () => '', + }) + .onCall(1) + .resolves({ + ok: true, + status: 200, + json: async () => ({ + results: [{ metadata: { room_id: 'allowed', msg_id: 'keyword-msg' }, text: 'keyword text', score: 0.4 }], + }), + text: async () => '', + }); + + const results = await createService().search({ + query: 'fruit', + userId: 'user-id', + searchType: 'hybrid', + limit: 5, + }); + + expect(serverFetch.callCount).to.equal(2); + expect(results).to.deep.equal([ + { + _id: 'allowed-msg', + rid: 'allowed', + msgId: 'allowed-msg', + text: 'allowed-msg from db', + ts: '2026-01-05T12:00:00.000Z', + u: { username: 'alice', name: 'Alice' }, + score: 0.8, + room: { _id: 'allowed', t: 'c', name: 'general', fname: 'General' }, + }, + { + _id: 'keyword-msg', + rid: 'allowed', + msgId: 'keyword-msg', + text: 'keyword-msg from db', + ts: '2026-01-05T12:00:00.000Z', + u: { username: 'alice', name: 'Alice' }, + score: 0.6, + room: { _id: 'allowed', t: 'c', name: 'general', fname: 'General' }, + }, + ]); + }); + + it('scales the candidate pool with the requested page and caps it', async () => { + serverFetch.resolves({ ok: true, status: 200, json: async () => ({ results: [] }), text: async () => '' }); + + const service = createService(); + await service.search({ query: 'fruit', userId: 'user-id', limit: 20 }); + expect(JSON.parse(serverFetch.lastCall.args[1].body).params.k).to.equal(60); + + await service.search({ query: 'fruit', userId: 'user-id', limit: 50 }); + expect(JSON.parse(serverFetch.lastCall.args[1].body).params.k).to.equal(100); + }); + + it('keeps a full page of hybrid results when fusion candidates are not visible', async () => { + cachedSettings.get.callsFake((key: string) => (key === 'AI_Intelligent_Search_Mode' ? 'hybrid' : settings[key])); + // only the last two candidates resolve to a visible message + Messages.findVisibleByIds.callsFake((msgIds: string[]) => + cursor( + msgIds + .filter((msgId) => msgId === 'visible-a' || msgId === 'visible-b') + .map((msgId) => ({ + _id: msgId, + rid: 'allowed', + msg: `${msgId} from db`, + ts: new Date('2026-01-05T12:00:00.000Z'), + u: { username: 'alice', name: 'Alice' }, + })), + ), + ); + const semanticResults = [ + { metadata: { room_id: 'allowed', msg_id: 'hidden-1' }, score: 0.2 }, + { metadata: { room_id: 'allowed', msg_id: 'hidden-2' }, score: 0.25 }, + { metadata: { room_id: 'allowed', msg_id: 'visible-a' }, score: 0.3 }, + ]; + const keywordResults = [{ metadata: { room_id: 'allowed', msg_id: 'visible-b' }, score: 0.4 }]; + serverFetch.reset(); + serverFetch + .onCall(0) + .resolves({ ok: true, status: 200, json: async () => ({ results: semanticResults }), text: async () => '' }) + .onCall(1) + .resolves({ ok: true, status: 200, json: async () => ({ results: keywordResults }), text: async () => '' }); + + const results = await createService().search({ query: 'fruit', userId: 'user-id', searchType: 'hybrid', limit: 2 }); + + expect(results.map(({ _id }: { _id: string }) => _id)).to.deep.equal(['visible-b', 'visible-a']); + }); + + it('promotes fresher messages once the recency boost is enabled', async () => { + const timestamps: Record = { + stale: '2020-01-01T12:00:00.000Z', + fresh: new Date().toISOString(), + }; + Messages.findVisibleByIds.callsFake((msgIds: string[]) => + cursor( + msgIds.map((msgId) => ({ + _id: msgId, + rid: 'allowed', + msg: `${msgId} from db`, + ts: new Date(timestamps[msgId]), + u: { username: 'alice', name: 'Alice' }, + })), + ), + ); + const pipelineResults = { + results: [ + { metadata: { room_id: 'allowed', msg_id: 'stale', timestamp: timestamps.stale }, score: 0.2 }, + { metadata: { room_id: 'allowed', msg_id: 'fresh', timestamp: timestamps.fresh }, score: 0.21 }, + ], + }; + serverFetch.resolves({ ok: true, status: 200, json: async () => pipelineResults, text: async () => '' }); + + const withoutBoost = await createService().search({ query: 'fruit', userId: 'user-id', limit: 5 }); + expect(withoutBoost.map(({ _id }: { _id: string }) => _id)).to.deep.equal(['stale', 'fresh']); + + cachedSettings.get.callsFake((key: string) => (key === 'AI_Intelligent_Search_Recency_Weight' ? 100 : settings[key])); + const withBoost = await createService().search({ query: 'fruit', userId: 'user-id', limit: 5 }); + expect(withBoost.map(({ _id }: { _id: string }) => _id)).to.deep.equal(['fresh', 'stale']); + }); + + it('falls back to a sane half-life when the setting is not usable', async () => { + cachedSettings.get.callsFake((key: string) => { + if (key === 'AI_Intelligent_Search_Recency_Weight') { + return 50; + } + if (key === 'AI_Intelligent_Search_Recency_Half_Life_Days') { + return 0; + } + + return settings[key]; + }); + serverFetch.resolves({ + ok: true, + status: 200, + json: async () => ({ results: [{ metadata: { room_id: 'allowed', msg_id: 'allowed-msg' }, score: 0.2 }] }), + text: async () => '', + }); + + const results = await createService().search({ query: 'fruit', userId: 'user-id', limit: 5 }); + + expect(results.map(({ _id }: { _id: string }) => _id)).to.deep.equal(['allowed-msg']); + }); + it('resolves room-name filters before querying the pipeline', async () => { Rooms.findOneByNameOrFname.resolves({ _id: 'room-general' }); Subscriptions.findByUserId.returns(cursor([{ rid: 'room-general' }])); diff --git a/docs/features/ai-search-hybrid-benchmark.md b/docs/features/ai-search-hybrid-benchmark.md new file mode 100644 index 0000000000000..9180153a849d3 --- /dev/null +++ b/docs/features/ai-search-hybrid-benchmark.md @@ -0,0 +1,101 @@ +# AI Search hybrid retrieval: offline benchmark + +Measurements behind the hybrid search defaults. Re-run these before changing +`INTELLIGENT_SEARCH_CANDIDATE_MULTIPLIER`, `MIN_INTELLIGENT_SEARCH_CANDIDATES`, +`INTELLIGENT_SEARCH_RRF_CONSTANT`, or the shipped setting defaults. + +## Method + +- **Corpus**: 547 synthetic Rocket.Chat messages ingested into a QA Intelligent Search pipeline — + 22 judged messages plus 525 topically adjacent distractors, so that top-k retrieval is actually + selective. Messages carry `room_id`, `username` and `timestamp` metadata exactly as production does. +- **Queries**: 20 judged queries in four families: + - `lexical` — exact identifiers (`CVE-2025-1337`, `SUP-4471`, `normalizeMessagesForUser`) + - `conceptual` — paraphrases with minimal lexical overlap with their targets + - `mixed` — an identifier plus a concept + - `recency` — two near-duplicate messages where only the newer one is correct +- **Grades**: 0-3 per (query, message). Metrics are nDCG@10, MRR@10 (first hit graded ≥ 2), recall@10. +- **Harness**: replicates `packages/ai-search/src/fusion.ts` exactly, including the `w = 0` / `w = 100` + short-circuits, and reuses one retrieval per branch across the whole sweep. + +Absolute numbers are only meaningful relative to each other: the corpus is synthetic and small. + +## Semantic weight sweep (candidate pool 50) + +| w | nDCG@10 | MRR@10 | R@10 | conceptual | lexical | mixed | recency | +| --- | --- | --- | --- | --- | --- | --- | --- | +| 0 (keyword only) | 0.3455 | 0.3000 | 0.450 | 0.0000 | 0.7891 | 0.2774 | 0.0000 | +| 10-40 | 0.6814 | 0.5892 | 0.933 | 0.6303 | 0.7891 | 0.6412 | 0.5582 | +| 50 | 0.7112 | 0.6392 | 0.933 | 0.6303 | 0.8418 | 0.6865 | 0.5582 | +| **60** | **0.7152** | 0.6392 | 0.933 | 0.6303 | 0.8418 | 0.7026 | 0.5582 | +| 70-90 | 0.7091 | 0.6392 | 0.933 | 0.6303 | 0.8418 | 0.6780 | 0.5582 | +| 100 (semantic only) | 0.7091 | 0.6392 | 0.933 | 0.6303 | 0.8418 | 0.6780 | 0.5582 | + +- Keyword-only is not a viable default: it scores **0.0** on conceptual and recency queries. +- Hybrid beats semantic-only, and the entire gain sits in `mixed` queries (0.7026 vs 0.6780, +3.6% + relative) — exactly the family hybrid exists for. Other families are unchanged. +- The curve is a step function rather than a smooth slope, because the keyword branch returns very few + rows (see the limitation below). The plateau from 50-90 means the setting is forgiving. + +**Shipped default: 50.** 60 measured marginally higher (+0.6% relative), which is well inside the noise +of a 20-query synthetic set. 50 is the neutral, defensible midpoint; revisit with judged production +queries rather than promoting 60 on this evidence. + +## Candidate pool sweep (w = 60) + +| candidate pool | best nDCG@10 | conceptual | +| --- | --- | --- | +| 20 | 0.6881 | 0.6014 | +| **50** | **0.7152** | 0.6303 | +| 100 | 0.6987 | 0.5677 | + +50 per branch is the peak. 20 starves fusion; 100 dilutes conceptual queries with weak neighbours. +Hence `MIN_INTELLIGENT_SEARCH_CANDIDATES = 50` — the default page size of 5 lands exactly on the +optimum, and larger pages scale by ×3 up to the cap of 100. + +## Temporal boost sweep (w = 60, candidate pool 50) + +nDCG@10, by recency weight and half-life: + +| recency weight | half-life 7d | half-life 30d | half-life 90d | +| --- | --- | --- | --- | +| 0 (off) | 0.7152 | 0.7152 | 0.7152 | +| 10 | 0.7422 | 0.7471 | 0.7296 | +| 25 | 0.7272 | **0.7507** | 0.7510 | +| 50 | 0.7012 | 0.7398 | 0.7495 | +| 75 | 0.6677 | 0.7339 | **0.7524** | +| 100 | 0.6581 | 0.7215 | 0.7505 | + +At weight 25 / half-life 30: overall nDCG@10 **0.7152 → 0.7507 (+5.0%)**, recency queries +**0.5582 → 0.7685 (+37.7%)**, and lexical queries are **completely unaffected** (0.8418 throughout) — +the boost never displaces an exact-identifier match. + +Aggressive settings are actively harmful: weight 100 with a 7-day half-life drops conceptual queries +from 0.6303 to 0.4695. Short half-lives are sharp and unforgiving; 30-90 days are stable. + +**Shipped default: weight 0 (disabled), half-life 30.** Hybrid relevance ships first and ranking stays +unchanged unless an admin opts in. Recommended starting point when enabling: **weight 25, half-life 30**. + +## Backend limitations found while benchmarking + +Both are pipeline-side and worth raising with the Intelligent Search team; neither is fixable in +Rocket.Chat. + +1. **Full-text search is strict AND.** `kubectl ramen` and `kubectl zzzznotaword` both return zero rows. + Any conversational multi-word query therefore returns nothing from the keyword branch, which is why + the keyword branch contributes to so few queries above. Keyword retrieval is a precision aid for + identifier-style queries, not a recall workhorse. +2. **Full-text recall is incomplete.** Probing every content token of every document against the index, + only **83%** (191/230) retrieved their own document. Misses include ordinary content words — + `webhook`, `stale`, `rate`, `connection`, `cluster`, `nodes`, `login`, `mobile`, `Safari` — and are + reproducible across re-ingestion of the same text, so they are not a one-off indexing glitch. + +If full-text recall improves, re-run the weight sweep: the keyword branch would carry far more weight +and the optimum would likely move. + +## Reproducing + +The harness is not checked in — it depends on live pipeline credentials. It ingests a generated corpus +via `POST /pipelines/{id}/documents`, queries `POST /pipelines/{id}/search` once per branch per query, +then replays `fusion.ts` locally across the parameter grid. Point it at a disposable pipeline; it writes +several hundred documents. diff --git a/docs/features/ai-search-hybrid.md b/docs/features/ai-search-hybrid.md new file mode 100644 index 0000000000000..5b4dff72c67ba --- /dev/null +++ b/docs/features/ai-search-hybrid.md @@ -0,0 +1,120 @@ +# AI Search: hybrid retrieval and temporal reranking + +## Overview + +AI Search retrieves messages from an external Intelligent Search pipeline. It supports three retrieval +modes, selected by the `AI_Intelligent_Search_Mode` setting and overridable per request via the +`searchType` query parameter on `GET /v1/ai.search`: + +| Mode | Pipeline request | Notes | +| --- | --- | --- | +| `semantic` (default) | `type: "similarity"`, `search_type: 2` | Vector retrieval. The only mode before this feature. | +| `keyword` | `type: "search"`, `search_type: 1` | Full-text retrieval. | +| `hybrid` | both, in parallel | Fused client-side with weighted RRF. | + +## Pipeline retrieval + +``` + QUERY + │ + Apply hard filters + room scope / username / date range + │ + ┌───────────┴───────────┐ + ▼ ▼ + Full-text search Semantic search + k = candidate pool k = candidate pool + │ │ + │ similarity guardrail + │ │ + └──────────┬────────────┘ + ▼ + Weighted RRF + │ + relevance + ▼ + Temporal boost + ▼ + visibility filtering + ▼ + Top N +``` + +Room scoping, username and date filters are applied by the pipeline itself +(`buildIntelligentSearchPipelineFilters`), so they constrain both branches identically. + +## Why fusion happens in Rocket.Chat, not in the pipeline + +The pipeline advertises a native `type: "hybrid"` placeholder, but it returns +`501 Hybrid placeholder is not implemented yet`, and its parameter schema exposes only `k` — there is no +weight. Client-side fusion is therefore both necessary today and the only way to offer an admin-tunable +balance. + +## Score conventions + +The two retrievers report scores on incompatible scales, verified empirically against a live pipeline: + +- Semantic `score` is a **cosine distance**: *lower is better*. The best hit for a well-matched query + scored `0.44`, an unrelated message `0.81`. +- Full-text `score` is a **rank**: *higher is better*. + +`normalizeIntelligentSearchCandidates` converts distance to similarity (`similarity = 1 - distance`) for +display and for the similarity guardrail. Fusion deliberately never compares the two raw scores — it +works on **rank positions only**, which is what makes the incompatible scales a non-problem. + +## Weighted RRF + +For a document `d`, with `w = AI_Intelligent_Search_Semantic_Weight / 100` and `C = 60`: + +``` +score(d) = (1 - w) / (C + rank_fulltext(d)) + w / (C + rank_semantic(d)) +``` + +A branch that did not return `d` contributes nothing. `w = 0` and `w = 100` short-circuit to a single +retriever, so the other branch is never even requested. + +`C` and the candidate pool size are implementation parameters and are intentionally **not** admin +settings. + +## The similarity guardrail + +`AI_Intelligent_Search_Min_Similarity_Percent` applies **only to semantic candidates**, and only after +retrieval. A keyword hit is never discarded for being semantically unremarkable — that is precisely the +case hybrid search exists to serve (exact error codes, ticket ids, function names). + +It defaults to `0` (disabled) and should stay that way for most workspaces: a fixed embedding threshold +is brittle across embedding models, query length, language and corpus, whereas ranking is stable. Treat +it as a garbage-result guardrail, not a quality control. + +## Temporal reranking + +Applied after fusion, so relevance selects the candidates and freshness only reorders them: + +``` +final(d) = score(d) × (1 + recencyWeight × 2^(-ageInDays / halfLifeDays)) +``` + +- `AI_Intelligent_Search_Recency_Weight` (0-100, default **0** = disabled). +- `AI_Intelligent_Search_Recency_Half_Life_Days` (default 30). + +Timestamps come from the pipeline fragment metadata, so the boost costs no extra database work. +Candidates without a usable timestamp keep their relevance score rather than being penalised. + +Because the boost is multiplicative and bounded by `1 + recencyWeight`, it can reorder near-ties but +cannot overturn a large relevance gap. + +## Candidate pool + +Each branch is asked for more candidates than the caller requested +(`limit × 3`, floored at 50, capped at 100). This exists for two reasons: + +1. fusion needs overlap to work with; +2. results are filtered for visibility and room subscription **after** retrieval, so a pool the size of + the page would return short pages. + +Neither the fusion nor the normalization step truncates before that filtering runs. + +## Benchmark + +See [ai-search-hybrid-benchmark.md](./ai-search-hybrid-benchmark.md) for the offline relevance +measurements behind the defaults. diff --git a/packages/ai-search/src/constants.ts b/packages/ai-search/src/constants.ts index 961aa578f2a49..1797c49f7b1bf 100644 --- a/packages/ai-search/src/constants.ts +++ b/packages/ai-search/src/constants.ts @@ -5,6 +5,15 @@ export const AI_SEARCH_RESULTS_PAGE_SIZE = 8; export const AI_SEARCH_FILTER_SUGGESTION_LIMIT = 5; export const AI_SEARCH_ROOM_LOOKUP_LIMIT = 20; export const MAX_INTELLIGENT_SEARCH_RESULTS = 50; +// Candidate pool retrieved from each retriever before fusion. Implementation detail, never exposed to +// admins. The floor of 50 is where offline nDCG@10 peaked: 20 starves fusion, 100 dilutes conceptual +// queries with weak neighbours. See docs/features/ai-search-hybrid-benchmark.md. +export const INTELLIGENT_SEARCH_CANDIDATE_MULTIPLIER = 3; +export const MIN_INTELLIGENT_SEARCH_CANDIDATES = 50; +export const MAX_INTELLIGENT_SEARCH_CANDIDATES = 100; +export const INTELLIGENT_SEARCH_RRF_CONSTANT = 60; +export const DEFAULT_INTELLIGENT_SEARCH_SEMANTIC_WEIGHT = 50; +export const DEFAULT_INTELLIGENT_SEARCH_RECENCY_HALF_LIFE_DAYS = 30; export const MAX_SEARCH_FILTER_VALUES = 25; export const MAX_ROOM_SEARCH_PATTERN_LENGTH = 64; export const MAX_AI_SERVICE_RESPONSE_SIZE = 5 * 1024 * 1024; diff --git a/packages/ai-search/src/fusion.spec.ts b/packages/ai-search/src/fusion.spec.ts new file mode 100644 index 0000000000000..bae5c079992fe --- /dev/null +++ b/packages/ai-search/src/fusion.spec.ts @@ -0,0 +1,201 @@ +import { INTELLIGENT_SEARCH_RRF_CONSTANT } from './constants'; +import { + applyTemporalRerank, + filterSemanticCandidatesByMinimumSimilarity, + fuseCandidatesWithWeightedRRF, + getRecencyDecay, + toRankedCandidates, +} from './fusion'; +import type { FusedIntelligentSearchCandidate, IntelligentSearchCandidate } from './types'; + +const candidate = (msgId: string, overrides: Partial = {}): IntelligentSearchCandidate => ({ + _id: msgId, + msgId, + rid: 'room', + pipelineText: `${msgId} text`, + ...overrides, +}); + +const ids = (candidates: Pick[]): (string | undefined)[] => candidates.map(({ msgId }) => msgId); + +describe('AI Search fusion helpers', () => { + describe('filterSemanticCandidatesByMinimumSimilarity', () => { + it('keeps every candidate when the guardrail is disabled', () => { + const candidates = [candidate('m1', { semanticSimilarity: 0.1 }), candidate('m2', { semanticSimilarity: 0.9 })]; + + expect(filterSemanticCandidatesByMinimumSimilarity(candidates, 0)).toEqual(candidates); + expect(filterSemanticCandidatesByMinimumSimilarity(candidates, Number.NaN)).toEqual(candidates); + }); + + it('drops candidates below the minimum similarity', () => { + const candidates = [ + candidate('m1', { semanticSimilarity: 0.82 }), + candidate('m2', { semanticSimilarity: 0.69 }), + candidate('m3', { semanticSimilarity: 0.7 }), + ]; + + expect(ids(filterSemanticCandidatesByMinimumSimilarity(candidates, 70))).toEqual(['m1', 'm3']); + }); + + it('keeps candidates that carry no semantic similarity, so keyword hits survive the guardrail', () => { + const candidates = [candidate('m1'), candidate('m2', { semanticSimilarity: 0.1 })]; + + expect(ids(filterSemanticCandidatesByMinimumSimilarity(candidates, 70))).toEqual(['m1']); + }); + }); + + describe('fuseCandidatesWithWeightedRRF', () => { + const semantic = [candidate('s1'), candidate('s2'), candidate('shared')]; + const keyword = [candidate('shared'), candidate('k1'), candidate('k2')]; + + it('fuses on rank only, so incompatible retriever score scales never meet', () => { + const highDistanceSemantic = [candidate('s1', { score: 0.02, semanticSimilarity: 0.02 })]; + const highScoreKeyword = [candidate('k1', { score: 0.99 })]; + + // the semantic branch wins purely because it outranks on its own list, not because 0.99 > 0.02 + expect(ids(fuseCandidatesWithWeightedRRF(highDistanceSemantic, highScoreKeyword, 90, 10))).toEqual(['s1', 'k1']); + }); + + it('rewards candidates returned by both retrievers', () => { + const fused = fuseCandidatesWithWeightedRRF(semantic, keyword, 50, 10); + + expect(fused[0].msgId).toBe('shared'); + expect(fused[0].semanticRank).toBe(3); + expect(fused[0].fulltextRank).toBe(1); + }); + + it('shifts the ordering as the balance moves towards semantic', () => { + expect(ids(fuseCandidatesWithWeightedRRF(semantic, keyword, 10, 3))).toEqual(['shared', 'k1', 'k2']); + // 'shared' still leads at 90: its keyword rank 1 tops up an otherwise last-place semantic rank 3 + expect(ids(fuseCandidatesWithWeightedRRF(semantic, keyword, 90, 3))).toEqual(['shared', 's1', 's2']); + }); + + it('applies the documented weighted RRF contribution', () => { + const [top] = fuseCandidatesWithWeightedRRF([candidate('only')], [], 60, 1); + const k = INTELLIGENT_SEARCH_RRF_CONSTANT; + + expect(top.rrfScore).toBeCloseTo(0.6 / (k + 1), 10); + }); + + it('degrades to the populated branch when the other retriever returns nothing', () => { + expect(ids(fuseCandidatesWithWeightedRRF(semantic, [], 50, 10))).toEqual(['s1', 's2', 'shared']); + expect(ids(fuseCandidatesWithWeightedRRF([], keyword, 50, 10))).toEqual(['shared', 'k1', 'k2']); + }); + + it('keeps the semantic similarity of a candidate found by both retrievers', () => { + const [top] = fuseCandidatesWithWeightedRRF( + [candidate('shared', { score: 0.83, semanticSimilarity: 0.83, semanticDistance: 0.17 })], + [candidate('shared')], + 50, + 1, + ); + + expect(top.score).toBe(0.83); + expect(top.semanticSimilarity).toBe(0.83); + }); + + it('clamps out-of-range weights and respects the limit', () => { + expect(ids(fuseCandidatesWithWeightedRRF(semantic, keyword, 500, 2))).toEqual(['s1', 's2']); + expect(ids(fuseCandidatesWithWeightedRRF(semantic, keyword, -20, 2))).toEqual(['shared', 'k1']); + }); + + it('ignores candidates without any usable identifier', () => { + const unidentified = { _id: '', msgId: '', rid: 'room', pipelineText: '' }; + + expect(fuseCandidatesWithWeightedRRF([unidentified], [], 50, 10)).toEqual([]); + }); + }); + + describe('toRankedCandidates', () => { + it('gives single-retriever results the same rank-based score shape as fusion', () => { + const ranked = toRankedCandidates([candidate('m1'), candidate('m2')]); + + expect(ranked[0].rrfScore).toBeCloseTo(1 / (INTELLIGENT_SEARCH_RRF_CONSTANT + 1), 10); + expect(ranked[0].semanticRank).toBe(1); + expect(ranked[1].semanticRank).toBe(2); + }); + + it('records the keyword rank for keyword-sourced candidates', () => { + const [ranked] = toRankedCandidates([candidate('m1', { source: 'keyword' })]); + + expect(ranked.fulltextRank).toBe(1); + expect(ranked.semanticRank).toBeUndefined(); + }); + }); + + describe('getRecencyDecay', () => { + it('halves the decay every half-life', () => { + expect(getRecencyDecay(0, 30)).toBe(1); + expect(getRecencyDecay(30, 30)).toBeCloseTo(0.5, 10); + expect(getRecencyDecay(60, 30)).toBeCloseTo(0.25, 10); + }); + + it('treats future timestamps as brand new and rejects an unusable half-life', () => { + expect(getRecencyDecay(-5, 30)).toBe(1); + expect(getRecencyDecay(10, 0)).toBe(0); + expect(getRecencyDecay(Number.NaN, 30)).toBe(0); + }); + }); + + describe('applyTemporalRerank', () => { + const now = new Date('2026-09-11T12:00:00.000Z'); + const daysAgo = (days: number): string => new Date(now.getTime() - days * 24 * 60 * 60 * 1000).toISOString(); + const ranked = (entries: [string, number, string][]): FusedIntelligentSearchCandidate[] => + entries.map(([msgId, rrfScore, ts]) => ({ ...candidate(msgId, { ts }), rrfScore })); + + it('is a no-op when the boost is disabled', () => { + const candidates = ranked([ + ['old', 0.02, daysAgo(400)], + ['new', 0.01, daysAgo(0)], + ]); + + expect(applyTemporalRerank(candidates, { recencyWeight: 0, halfLifeDays: 30, now })).toEqual(candidates); + }); + + it('promotes a fresher candidate over a slightly more relevant stale one', () => { + const candidates = ranked([ + ['stale', 0.016, daysAgo(365)], + ['fresh', 0.015, daysAgo(0)], + ]); + + expect(ids(applyTemporalRerank(candidates, { recencyWeight: 100, halfLifeDays: 30, now }))).toEqual(['fresh', 'stale']); + }); + + it('cannot overturn a large relevance gap', () => { + const candidates = ranked([ + ['relevant', 0.05, daysAgo(365)], + ['fresh', 0.015, daysAgo(0)], + ]); + + expect(ids(applyTemporalRerank(candidates, { recencyWeight: 100, halfLifeDays: 30, now }))).toEqual(['relevant', 'fresh']); + }); + + it('leaves candidates without a usable timestamp at their relevance score rather than penalising them', () => { + const candidates: FusedIntelligentSearchCandidate[] = [ + { ...candidate('no-ts'), rrfScore: 0.02 }, + { ...candidate('bad-ts', { ts: 'not-a-date' }), rrfScore: 0.019 }, + { ...candidate('fresh', { ts: daysAgo(0) }), rrfScore: 0.018 }, + ]; + + expect(ids(applyTemporalRerank(candidates, { recencyWeight: 50, halfLifeDays: 30, now }))).toEqual(['fresh', 'no-ts', 'bad-ts']); + }); + + it('keeps the incoming order for ties', () => { + const candidates = ranked([ + ['first', 0.02, daysAgo(10)], + ['second', 0.02, daysAgo(10)], + ]); + + expect(ids(applyTemporalRerank(candidates, { recencyWeight: 40, halfLifeDays: 30, now }))).toEqual(['first', 'second']); + }); + + it('ignores an unusable half-life instead of dropping the boost silently', () => { + const candidates = ranked([ + ['stale', 0.016, daysAgo(365)], + ['fresh', 0.015, daysAgo(0)], + ]); + + expect(ids(applyTemporalRerank(candidates, { recencyWeight: 100, halfLifeDays: 0, now }))).toEqual(['stale', 'fresh']); + }); + }); +}); diff --git a/packages/ai-search/src/fusion.ts b/packages/ai-search/src/fusion.ts new file mode 100644 index 0000000000000..b7eaa03a65f17 --- /dev/null +++ b/packages/ai-search/src/fusion.ts @@ -0,0 +1,163 @@ +import { INTELLIGENT_SEARCH_RRF_CONSTANT } from './constants'; +import type { + FusedIntelligentSearchCandidate, + IntelligentSearchCandidate, + IntelligentSearchCandidateSource, + TemporalRerankOptions, +} from './types'; + +const clampPercent = (value: unknown): number => { + const numeric = Number(value); + if (!Number.isFinite(numeric)) { + return 0; + } + + return Math.min(100, Math.max(0, Math.floor(numeric))); +}; + +const getCandidateId = (candidate: IntelligentSearchCandidate): string => candidate.msgId || candidate._id; + +/** + * Drops semantic candidates whose similarity is below the configured guardrail. Keyword candidates are + * never scored by the embedding model, so the guardrail deliberately does not apply to them: an exact + * error code or ticket id must not be discarded because it is semantically unremarkable. + */ +export const filterSemanticCandidatesByMinimumSimilarity = ( + candidates: IntelligentSearchCandidate[], + minimumSimilarityPercent: number, +): IntelligentSearchCandidate[] => { + const minimumSimilarity = clampPercent(minimumSimilarityPercent); + if (!minimumSimilarity) { + return candidates; + } + + const threshold = minimumSimilarity / 100; + return candidates.filter((candidate) => candidate.semanticSimilarity === undefined || candidate.semanticSimilarity >= threshold); +}; + +/** + * Weighted Reciprocal Rank Fusion. + * + * The two retrievers report scores on incompatible scales (the pipeline returns cosine *distance* for + * semantic hits and a full-text rank for keyword hits), so fusion works on ranks only and the raw scores + * never meet. `semanticWeight` is the admin-facing 0-100 balance: 0 is keyword-only, 100 semantic-only. + */ +export const fuseCandidatesWithWeightedRRF = ( + semanticCandidates: IntelligentSearchCandidate[], + keywordCandidates: IntelligentSearchCandidate[], + semanticWeight: number, + limit: number, + rrfConstant: number = INTELLIGENT_SEARCH_RRF_CONSTANT, +): FusedIntelligentSearchCandidate[] => { + const normalizedSemanticWeight = clampPercent(semanticWeight) / 100; + const normalizedKeywordWeight = 1 - normalizedSemanticWeight; + + const fused = new Map(); + const addBranch = (candidates: IntelligentSearchCandidate[], branch: IntelligentSearchCandidateSource, branchWeight: number): void => { + for (let index = 0; index < candidates.length; index++) { + const candidate = candidates[index]; + const candidateId = getCandidateId(candidate); + if (!candidateId) { + continue; + } + + const rank = index + 1; + const contribution = branchWeight / (rrfConstant + rank); + const existing = fused.get(candidateId); + if (!existing) { + fused.set(candidateId, { + ...candidate, + rrfScore: contribution, + ...(branch === 'semantic' ? { semanticRank: rank } : { fulltextRank: rank }), + }); + continue; + } + + fused.set(candidateId, { + // a candidate found by both retrievers keeps the semantic similarity for display + ...existing, + ...(branch === 'semantic' && { + score: candidate.score ?? existing.score, + semanticSimilarity: candidate.semanticSimilarity ?? existing.semanticSimilarity, + semanticDistance: candidate.semanticDistance ?? existing.semanticDistance, + }), + ts: existing.ts || candidate.ts, + rrfScore: existing.rrfScore + contribution, + ...(branch === 'semantic' ? { semanticRank: rank } : { fulltextRank: rank }), + }); + } + }; + + addBranch(semanticCandidates, 'semantic', normalizedSemanticWeight); + addBranch(keywordCandidates, 'keyword', normalizedKeywordWeight); + + return [...fused.values()] + .sort((a, b) => { + if (b.rrfScore !== a.rrfScore) { + return b.rrfScore - a.rrfScore; + } + + const semanticRankDelta = (a.semanticRank ?? Number.MAX_SAFE_INTEGER) - (b.semanticRank ?? Number.MAX_SAFE_INTEGER); + if (semanticRankDelta !== 0) { + return semanticRankDelta; + } + + return (a.fulltextRank ?? Number.MAX_SAFE_INTEGER) - (b.fulltextRank ?? Number.MAX_SAFE_INTEGER); + }) + .slice(0, limit); +}; + +/** + * Turns an already relevance-ordered list into fused candidates so that every retrieval mode reaches the + * temporal stage with a comparable rank-based score. + */ +export const toRankedCandidates = ( + candidates: IntelligentSearchCandidate[], + rrfConstant: number = INTELLIGENT_SEARCH_RRF_CONSTANT, +): FusedIntelligentSearchCandidate[] => + candidates.map((candidate, index) => ({ + ...candidate, + rrfScore: 1 / (rrfConstant + index + 1), + ...(candidate.source === 'keyword' ? { fulltextRank: index + 1 } : { semanticRank: index + 1 }), + })); + +export const getRecencyDecay = (ageInDays: number, halfLifeDays: number): number => { + if (!(halfLifeDays > 0) || !Number.isFinite(ageInDays)) { + return 0; + } + + return 2 ** (-Math.max(0, ageInDays) / halfLifeDays); +}; + +/** + * Multiplicative temporal boost applied after relevance fusion. A candidate posted right now scores + * `1 + recencyWeight` times its relevance, decaying by half every `halfLifeDays`. Candidates without a + * usable timestamp are left untouched rather than penalised, so a missing `ts` can never demote a hit. + */ +export const applyTemporalRerank = ( + candidates: FusedIntelligentSearchCandidate[], + { recencyWeight, halfLifeDays, now = new Date() }: TemporalRerankOptions, +): FusedIntelligentSearchCandidate[] => { + const normalizedRecencyWeight = clampPercent(recencyWeight) / 100; + if (!normalizedRecencyWeight || !(halfLifeDays > 0)) { + return candidates; + } + + const nowMs = now.getTime(); + const millisecondsPerDay = 24 * 60 * 60 * 1000; + + return candidates + .map((candidate, index) => { + const timestamp = candidate.ts ? new Date(candidate.ts).getTime() : Number.NaN; + if (Number.isNaN(timestamp)) { + return { candidate, index, score: candidate.rrfScore }; + } + + const ageInDays = (nowMs - timestamp) / millisecondsPerDay; + const decay = getRecencyDecay(ageInDays, halfLifeDays); + + return { candidate, index, score: candidate.rrfScore * (1 + normalizedRecencyWeight * decay) }; + }) + .sort((a, b) => (b.score !== a.score ? b.score - a.score : a.index - b.index)) + .map(({ candidate }) => candidate); +}; diff --git a/packages/ai-search/src/index.ts b/packages/ai-search/src/index.ts index e3f7bd0f029af..effd4e72f66a6 100644 --- a/packages/ai-search/src/index.ts +++ b/packages/ai-search/src/index.ts @@ -1,5 +1,6 @@ export * from './clientSearch'; export * from './constants'; +export * from './fusion'; export * from './intelligentSearch'; export * from './llm'; export type * from './types'; diff --git a/packages/ai-search/src/intelligentSearch.spec.ts b/packages/ai-search/src/intelligentSearch.spec.ts index 5b917c447a504..627ee1d2d5a2b 100644 --- a/packages/ai-search/src/intelligentSearch.spec.ts +++ b/packages/ai-search/src/intelligentSearch.spec.ts @@ -43,10 +43,37 @@ describe('AI Search intelligent search helpers', () => { ); expect(results).toEqual([ - { _id: 'm1', rid: 'r1', msgId: 'm1', pipelineText: 'metadata text', score: 0.89 }, - { _id: 'm2', rid: 'r2', msgId: 'm2', pipelineText: 'content text', score: 0.49 }, - { _id: 'm3', rid: 'r3', msgId: 'm3', pipelineText: 'document text', score: 0.88 }, - { _id: 'm4', rid: 'r4', msgId: 'm4', pipelineText: 'no numeric score' }, + { + _id: 'm1', + rid: 'r1', + msgId: 'm1', + pipelineText: 'metadata text', + score: 0.89, + semanticSimilarity: 0.89, + semanticDistance: 0.11, + source: 'semantic', + }, + { + _id: 'm2', + rid: 'r2', + msgId: 'm2', + pipelineText: 'content text', + score: 0.49, + semanticSimilarity: 0.49, + semanticDistance: 0.51, + source: 'semantic', + }, + { + _id: 'm3', + rid: 'r3', + msgId: 'm3', + pipelineText: 'document text', + score: 0.88, + semanticSimilarity: 0.88, + semanticDistance: 0.12, + source: 'semantic', + }, + { _id: 'm4', rid: 'r4', msgId: 'm4', pipelineText: 'no numeric score', source: 'semantic' }, ]); }); @@ -58,7 +85,7 @@ describe('AI Search intelligent search helpers', () => { expect(normalizeIntelligentSearchCandidates(rawResults, [], 10)).toHaveLength(2); expect(normalizeIntelligentSearchCandidates(rawResults, ['allowed'], 10)).toEqual([ - { _id: 'm1', rid: 'allowed', msgId: 'm1', pipelineText: 'allowed' }, + { _id: 'm1', rid: 'allowed', msgId: 'm1', pipelineText: 'allowed', source: 'semantic' }, ]); }); @@ -69,7 +96,50 @@ describe('AI Search intelligent search helpers', () => { 1, ); - expect(results).toEqual([{ _id: 'm1', rid: 'r1', msgId: 'm1', pipelineText: '' }]); + expect(results).toEqual([{ _id: 'm1', rid: 'r1', msgId: 'm1', pipelineText: '', source: 'semantic' }]); + }); + + it('optionally marks the semantic source for caller-provided source input', () => { + expect( + normalizeIntelligentSearchCandidates( + { results: [{ metadata: { room_id: 'r1', msg_id: 'm1' }, text: 'keyword match', score: 0.42 }] }, + ['r1'], + 10, + undefined, + 'keyword', + ), + ).toEqual([ + { + _id: 'm1', + rid: 'r1', + msgId: 'm1', + pipelineText: 'keyword match', + score: 0.58, + semanticSimilarity: 0.58, + semanticDistance: 0.42, + source: 'keyword', + }, + ]); + }); + }); + + describe('candidate timestamps', () => { + it('carries the pipeline timestamp through for the temporal rerank stage', () => { + const [withMetadataTs, withResultTs, withoutTs] = normalizeIntelligentSearchCandidates( + { + results: [ + { metadata: { room_id: 'r1', msg_id: 'm1', timestamp: '2026-01-05T12:00:00.000Z' } }, + { metadata: { room_id: 'r2', msg_id: 'm2' }, timestamp: '2026-02-05T12:00:00.000Z' }, + { metadata: { room_id: 'r3', msg_id: 'm3' } }, + ], + }, + [], + 10, + ); + + expect(withMetadataTs.ts).toBe('2026-01-05T12:00:00.000Z'); + expect(withResultTs.ts).toBe('2026-02-05T12:00:00.000Z'); + expect(withoutTs).not.toHaveProperty('ts'); }); }); @@ -167,6 +237,49 @@ describe('AI Search intelligent search helpers', () => { }); }); + it('requests keyword mode and omits threshold when semantic filtering is not applicable', async () => { + let requestBody = ''; + const fetch: AIServiceFetch = async (_url, options) => { + requestBody = String(options.body); + + return { + ok: true, + status: 200, + json: async () => ({ results: [] }), + text: async () => '', + }; + }; + + await searchIntelligentPipeline({ + query: 'service health', + config: { + baseUrl: 'https://pipeline.example.com/', + pipelineId: 'workspace', + apiKey: 'key', + apiKeySecret: 'secret', + minimumSimilarityPercent: 70, + }, + classifications: ['user'], + pipelineFilters: { room_id: { $in: ['r1'] } }, + limit: 5, + fetch, + mode: 'keyword', + }); + + expect(JSON.parse(requestBody)).toEqual({ + query: 'service health', + type: 'search', + classification: { + classifications: ['user'], + search_type: 1, + }, + filters: { room_id: { $in: ['r1'] } }, + params: { + k: 5, + }, + }); + }); + it('returns an empty result set for non-2xx pipeline responses', async () => { const fetch: AIServiceFetch = async () => ({ ok: false, diff --git a/packages/ai-search/src/intelligentSearch.ts b/packages/ai-search/src/intelligentSearch.ts index 7016472936e5f..a8956f93361fb 100644 --- a/packages/ai-search/src/intelligentSearch.ts +++ b/packages/ai-search/src/intelligentSearch.ts @@ -3,6 +3,7 @@ import type { AIServiceFetch, AIServiceLogger, IntelligentSearchCandidate, + IntelligentSearchCandidateSource, IntelligentSearchFilters, IntelligentSearchPipelineFilters, IntelligentSearchPipelineRequest, @@ -55,26 +56,42 @@ export const normalizeSimilarityPercent = (value: unknown): number => { export const getSemanticDistanceThreshold = (minimumSimilarityPercent: number): number => Number((1 - minimumSimilarityPercent / 100).toFixed(4)); -// pipeline contract: `score`/`distance` are cosine distances (lower is better, similarity = 1 - distance) -const normalizePipelineSimilarityScore = (value: number, type: 'distance' | 'similarity'): number => { +// pipeline contract (verified against the Intelligent Search API): +// - `score`/`distance` are cosine *distances* - lower is better, in [0,1] +// - `similarity` values are cosine *similarities* - higher is better, in [0,1] +// Both are clamped to [0,1]; percentages are accepted for resilience against provider drift. +const normalizePipelineScore = (value: number): number => { const normalizedValue = Math.abs(value) > 1 ? value / 100 : value; - const similarity = type === 'distance' ? 1 - normalizedValue : normalizedValue; - return Math.min(1, Math.max(0, similarity)); + return Math.min(1, Math.max(0, normalizedValue)); }; -const extractPipelineSimilarityScore = (result: Record, metadata: Record): number | undefined => { +const extractPipelineSimilarityScores = ( + result: Record, + metadata: Record, +): { semanticSimilarity?: number; semanticDistance?: number } => { const similarity = firstNumber(result.similarity, metadata.similarity); if (typeof similarity === 'number') { - return normalizePipelineSimilarityScore(similarity, 'similarity'); + const semanticSimilarity = normalizePipelineScore(similarity); + const semanticDistance = Number((1 - semanticSimilarity).toFixed(4)); + + return { + semanticSimilarity, + semanticDistance, + }; } const distance = firstNumber(result.score, result.distance, metadata.score, metadata.distance); if (typeof distance === 'number') { - return normalizePipelineSimilarityScore(distance, 'distance'); + const semanticDistance = normalizePipelineScore(distance); + + return { + semanticSimilarity: Number((1 - semanticDistance).toFixed(4)), + semanticDistance, + }; } - return undefined; + return {}; }; const extractIntelligentResultIds = (result: Record): { rid?: string; msgId?: string } => { @@ -101,6 +118,7 @@ export const normalizeIntelligentSearchCandidates = ( userRoomIds: string[] = [], limit: number, logger?: AIServiceLogger, + source: IntelligentSearchCandidateSource = 'semantic', ): IntelligentSearchCandidate[] => { let rawResults: unknown[] = []; const rawSearchResultsRecord = asRecord(rawSearchResults); @@ -141,13 +159,20 @@ export const normalizeIntelligentSearchCandidates = ( continue; } - const score = extractPipelineSimilarityScore(result, metadata); + const { semanticDistance, semanticSimilarity } = extractPipelineSimilarityScores(result, metadata); + const ts = firstString(metadata.timestamp, result.timestamp); candidates.push({ _id: msgId || `intelligent-${index}`, rid, msgId, pipelineText: firstString(result.text, result.content, result.document, result.page_content, metadata.text) || '', - ...(typeof score === 'number' && { score }), + ...(ts && { ts }), + ...(typeof semanticSimilarity === 'number' && { + score: semanticSimilarity, + semanticSimilarity, + semanticDistance, + }), + ...(source && { source }), }); } @@ -228,19 +253,24 @@ export const searchIntelligentPipeline = async ({ limit, fetch, logger, + mode = 'semantic', }: IntelligentSearchPipelineRequest): Promise => { const minimumSimilarity = normalizeSimilarityPercent(config.minimumSimilarityPercent); const formattedQuery = config.queryTemplate ? config.queryTemplate.replace('{query}', query) : query; const url = buildEndpointUrl(config.baseUrl, `pipelines/${encodeURIComponent(config.pipelineId)}/search`); + const searchType = mode === 'keyword' ? 'search' : 'similarity'; + const shouldApplyThreshold = mode !== 'keyword'; + const threshold = shouldApplyThreshold ? getSemanticDistanceThreshold(minimumSimilarity) : undefined; logger?.debug?.({ msg: 'Intelligent search request', url, queryLength: formattedQuery.length, hasQueryTemplate: Boolean(config.queryTemplate), + searchType, filterKeys: Object.keys(pipelineFilters), classificationCount: classifications.length, - threshold: getSemanticDistanceThreshold(minimumSimilarity), + threshold, }); let response: Awaited>; @@ -257,15 +287,15 @@ export const searchIntelligentPipeline = async ({ }, body: JSON.stringify({ query: formattedQuery, - type: 'similarity', + type: searchType, classification: { classifications, - search_type: 2, + search_type: mode === 'keyword' ? 1 : 2, }, filters: pipelineFilters, params: { k: limit, - threshold: getSemanticDistanceThreshold(minimumSimilarity), + ...(typeof threshold === 'number' && { threshold }), }, }), }); diff --git a/packages/ai-search/src/types.ts b/packages/ai-search/src/types.ts index f98dd1245ba7e..c4943cd02442e 100644 --- a/packages/ai-search/src/types.ts +++ b/packages/ai-search/src/types.ts @@ -50,6 +50,36 @@ export type IntelligentSearchPipelineConfig = { minimumSimilarityPercent?: number; }; +export type IntelligentSearchType = 'semantic' | 'keyword' | 'hybrid'; + +export type IntelligentSearchCandidateSource = 'semantic' | 'keyword'; + +export type IntelligentSearchCandidate = { + _id: string; + rid?: string; + msgId?: string; + pipelineText: string; + score?: number; + semanticSimilarity?: number; + semanticDistance?: number; + source?: IntelligentSearchCandidateSource; + /** message timestamp reported by the pipeline, used by the temporal rerank stage */ + ts?: string; +}; + +export type FusedIntelligentSearchCandidate = IntelligentSearchCandidate & { + rrfScore: number; + semanticRank?: number; + fulltextRank?: number; +}; + +export type TemporalRerankOptions = { + /** 0 disables the boost, 100 gives the freshest candidate double the relevance score */ + recencyWeight: number; + halfLifeDays: number; + now?: Date; +}; + export type IntelligentSearchFilters = { rid?: string; rids?: string[]; @@ -62,14 +92,6 @@ export type IntelligentSearchFilters = { export type IntelligentSearchPipelineFilters = Record; -export type IntelligentSearchCandidate = { - _id: string; - rid?: string; - msgId?: string; - pipelineText: string; - score?: number; -}; - export type IntelligentSearchPipelineRequest = { query: string; config: IntelligentSearchPipelineConfig; @@ -78,4 +100,5 @@ export type IntelligentSearchPipelineRequest = { limit: number; fetch: AIServiceFetch; logger?: AIServiceLogger; + mode?: IntelligentSearchType; }; diff --git a/packages/core-services/src/types/IAISearchService.ts b/packages/core-services/src/types/IAISearchService.ts index c670d5e8d0a93..76bfddc9ab9f1 100644 --- a/packages/core-services/src/types/IAISearchService.ts +++ b/packages/core-services/src/types/IAISearchService.ts @@ -19,6 +19,8 @@ export type AISearchStatus = { answerGenerationConfigured: boolean; }; +export type AISearchType = 'semantic' | 'keyword' | 'hybrid'; + export type AISearchAnswerMessage = { text: string; username?: string; @@ -54,7 +56,13 @@ export type AISearchResult = { export interface IAISearchService extends IServiceClass { status(): Promise; - search(params: { query: string; userId: string; filters?: AISearchFilters; limit?: number }): Promise; + search(params: { + query: string; + userId: string; + filters?: AISearchFilters; + limit?: number; + searchType?: AISearchType; + }): Promise; answer(params: { query: string; messages: AISearchAnswerMessage[] }): Promise; diff --git a/packages/i18n/src/locales/en.i18n.json b/packages/i18n/src/locales/en.i18n.json index b3fb8c78dc088..c3f5f7cd8323f 100644 --- a/packages/i18n/src/locales/en.i18n.json +++ b/packages/i18n/src/locales/en.i18n.json @@ -582,12 +582,23 @@ "AI_Intelligent_Search_Enabled_Description": "Use the configured vector-search pipeline to add semantic results to workspace search.", "AI_Intelligent_Search_Min_Similarity_Percent": "Minimum semantic similarity (%)", "AI_Intelligent_Search_Min_Similarity_Percent_Description": "Higher values return fewer but closer semantic matches. Use 0 to keep the widest result set.", + "AI_Intelligent_Search_Mode": "Search method", + "AI_Intelligent_Search_Mode_Description": "How AI Search retrieves messages. Hybrid combines keyword and semantic retrieval.", + "AI_Intelligent_Search_Mode_Hybrid": "Hybrid", + "AI_Intelligent_Search_Mode_Keyword": "Keyword only", + "AI_Intelligent_Search_Mode_Semantic": "Semantic only", "AI_Intelligent_Search_Pipeline_Base_URL": "Pipeline API base URL", "AI_Intelligent_Search_Pipeline_Base_URL_Description": "Base URL for the intelligent-search pipeline API.", "AI_Intelligent_Search_Pipeline_ID": "Pipeline ID", "AI_Intelligent_Search_Pipeline_ID_Description": "Identifier of the target pipeline used for semantic search requests.", "AI_Intelligent_Search_Query_Template": "Query template", "AI_Intelligent_Search_Query_Template_Description": "Optional template applied to search queries before sending to the pipeline. Use {query} as the placeholder. Leave blank to send the raw query.", + "AI_Intelligent_Search_Recency_Half_Life_Days": "Recency half-life (days)", + "AI_Intelligent_Search_Recency_Half_Life_Days_Description": "Age at which a message keeps half of its recency boost.", + "AI_Intelligent_Search_Recency_Weight": "Recency boost", + "AI_Intelligent_Search_Recency_Weight_Description": "How strongly newer messages are promoted after relevance ranking. Use 0 to rank purely by relevance.", + "AI_Intelligent_Search_Semantic_Weight": "Hybrid search balance", + "AI_Intelligent_Search_Semantic_Weight_Description": "Relative influence of semantic retrieval in hybrid mode: 0 is keyword only, 100 is semantic only. This is not the percentage of results that come from semantic search.", "AI_LLM_OpenAI_API_Key": "API key", "AI_LLM_OpenAI_API_Key_Description": "API key for the OpenAI-compatible chat completions endpoint.", "AI_LLM_OpenAI_Base_URL": "API base URL", diff --git a/packages/rest-typings/src/v1/aiSearch.ts b/packages/rest-typings/src/v1/aiSearch.ts index 8d36c9eeb4527..bdfb46af16b66 100644 --- a/packages/rest-typings/src/v1/aiSearch.ts +++ b/packages/rest-typings/src/v1/aiSearch.ts @@ -12,6 +12,7 @@ type AISearch = { fromUsernames?: string; startDate?: string; endDate?: string; + searchType?: 'semantic' | 'keyword' | 'hybrid'; }; const AISearchSchema = { @@ -38,6 +39,7 @@ const AISearchSchema = { { type: 'string', format: 'date-time' }, ], }, + searchType: { type: 'string', enum: ['semantic', 'keyword', 'hybrid'] }, }, required: ['query'], additionalProperties: false, From 341dbbd222667eb1382b6d572a2c1342bc0fc376 Mon Sep 17 00:00:00 2001 From: Dnouv Date: Fri, 11 Sep 2026 22:08:13 +0800 Subject: [PATCH 02/10] refactor: collapse AI Search retrieval settings into one balance The search-mode select was redundant: the 0-100 balance already expresses every mode, since 0 and 100 short-circuit to a single retriever. Two controls could only ever disagree with each other. `searchType` stays on the REST endpoint so a caller can still pin an endpoint of the range per request. Drops the recency half-life setting too. Offline sweeps put the 30-day and 90-day curves within 0.3% nDCG of each other across the useful weight range, so the knob bought no reachable quality; it is now a constant. Four new admin settings become two: search balance and recency boost. Also retunes the candidate pool after measuring pipeline latency, which turned up a regression in the previous commit: raising the pool to 50 slowed the *default* semantic path from 588ms to 910ms p50, on every debounced keystroke in the navbar. A pool of 20 costs what the old pool of 5 cost (584ms) while lifting nDCG@10 by 6.9%, and 100 measured both slower and worse than 50. Floor is now 20, cap 50. --- .changeset/hybrid-ai-search-retrieval.md | 12 +-- .../server/services/ai-search/service.ts | 49 ++++------- apps/meteor/server/settings/ai.ts | 31 +------ .../services/ai-search/service.tests.ts | 84 +++++++------------ docs/features/ai-search-hybrid-benchmark.md | 50 ++++++++--- docs/features/ai-search-hybrid.md | 33 ++++++-- packages/ai-search/src/constants.ts | 10 ++- packages/i18n/src/locales/en.i18n.json | 13 +-- 8 files changed, 125 insertions(+), 157 deletions(-) diff --git a/.changeset/hybrid-ai-search-retrieval.md b/.changeset/hybrid-ai-search-retrieval.md index 9f06f1120c8c5..cb9dfc8f1d03d 100644 --- a/.changeset/hybrid-ai-search-retrieval.md +++ b/.changeset/hybrid-ai-search-retrieval.md @@ -8,13 +8,13 @@ Adds hybrid retrieval and optional temporal reranking to AI Search. -A new **Search method** setting selects semantic, keyword, or hybrid retrieval. In hybrid mode the -semantic and full-text retrievers run in parallel and are fused with weighted Reciprocal Rank Fusion, -balanced by a **Hybrid search balance** setting (0 is keyword only, 100 is semantic only). Fusion works -on rank positions, so the retrievers' incompatible score scales are never compared directly. +A single **Search balance** setting (0-100) now controls retrieval: 0 searches by keyword only, 100 by +meaning only, and anything in between runs both retrievers in parallel and fuses them with weighted +Reciprocal Rank Fusion. Fusion works on rank positions, so the retrievers' incompatible score scales are +never compared directly. The minimum semantic similarity guardrail now applies only to semantic candidates, so an exact match on an error code or ticket id is no longer discarded for being semantically unremarkable. -An optional **Recency boost** reranks results by age after relevance ranking, using an exponential -half-life decay. It is disabled by default and leaves ranking unchanged until an admin opts in. +An optional **Recency boost** reranks results by age after relevance ranking, using an exponential decay +with a 30-day half-life. It is disabled by default and leaves ranking unchanged until an admin opts in. diff --git a/apps/meteor/server/services/ai-search/service.ts b/apps/meteor/server/services/ai-search/service.ts index f5ea6a687a2c3..b7e831ec578a9 100644 --- a/apps/meteor/server/services/ai-search/service.ts +++ b/apps/meteor/server/services/ai-search/service.ts @@ -167,24 +167,19 @@ export class AISearchService extends ServiceClass implements IAISearchService { }; } - private getSearchMode(): IntelligentSearchType { - const configuredMode = settings.get('AI_Intelligent_Search_Mode'); - if (configuredMode === 'hybrid' || configuredMode === 'keyword' || configuredMode === 'semantic') { - return configuredMode; + /** + * The 0-100 balance is the whole retrieval control: 0 is keyword only, 100 is semantic only, anything + * between fuses both. A per-request `searchType` pins an endpoint of that range without an admin change. + */ + private resolveSemanticWeight(searchType: IntelligentSearchType | undefined): number { + if (searchType === 'keyword') { + return 0; } - return 'semantic'; - } - - private normalizeSearchType(searchType: IntelligentSearchType | undefined): IntelligentSearchType { - if (searchType === 'hybrid' || searchType === 'keyword' || searchType === 'semantic') { - return searchType; + if (searchType === 'semantic') { + return 100; } - return this.getSearchMode(); - } - - private getHybridWeight(): number { const configuredWeight = Number(settings.get('AI_Intelligent_Search_Semantic_Weight')); if (!Number.isFinite(configuredWeight)) { return DEFAULT_INTELLIGENT_SEARCH_SEMANTIC_WEIGHT; @@ -202,15 +197,6 @@ export class AISearchService extends ServiceClass implements IAISearchService { return Math.min(100, Math.max(0, Math.floor(configuredWeight))); } - private getRecencyHalfLifeDays(): number { - const configuredHalfLife = Number(settings.get('AI_Intelligent_Search_Recency_Half_Life_Days')); - if (!Number.isFinite(configuredHalfLife) || configuredHalfLife <= 0) { - return DEFAULT_INTELLIGENT_SEARCH_RECENCY_HALF_LIFE_DAYS; - } - - return Math.floor(configuredHalfLife); - } - private async queryPipelineCandidates({ query, config, @@ -246,21 +232,20 @@ export class AISearchService extends ServiceClass implements IAISearchService { classifications: string[], pipelineFilters: IntelligentSearchPipelineFilters, limit: number, - searchMode: IntelligentSearchType, + semanticWeight: number, ): Promise { const candidateLimit = this.getSearchCandidateLimit(limit); const queryBranch = (sourceMode: 'semantic' | 'keyword') => this.queryPipelineCandidates({ query, config, classifications, pipelineFilters, limit: candidateLimit, sourceMode }); const minimumSimilarityPercent = Number(config.minimumSimilarityPercent || 0); - const semanticWeight = searchMode === 'hybrid' ? this.getHybridWeight() : undefined; - // a hybrid search collapses to a single retriever at the extremes of the balance slider - if (searchMode === 'keyword' || semanticWeight === 0) { + // at the extremes only one retriever is worth paying for, so the other is never requested + if (semanticWeight === 0) { return toRankedCandidates(await queryBranch('keyword')); } - if (searchMode === 'semantic' || semanticWeight === 100) { + if (semanticWeight === 100) { return toRankedCandidates(filterSemanticCandidatesByMinimumSimilarity(await queryBranch('semantic'), minimumSimilarityPercent)); } @@ -269,7 +254,7 @@ export class AISearchService extends ServiceClass implements IAISearchService { return fuseCandidatesWithWeightedRRF( filterSemanticCandidatesByMinimumSimilarity(semanticCandidates, minimumSimilarityPercent), keywordCandidates, - semanticWeight ?? DEFAULT_INTELLIGENT_SEARCH_SEMANTIC_WEIGHT, + semanticWeight, candidateLimit, ); } @@ -498,12 +483,12 @@ export class AISearchService extends ServiceClass implements IAISearchService { return []; } - const requestedMode = this.normalizeSearchType(searchType); - const candidates = await this.buildSearchCandidatesForMode(query, config, classifications, pipelineFilters, limit, requestedMode); + const semanticWeight = this.resolveSemanticWeight(searchType); + const candidates = await this.buildSearchCandidatesForMode(query, config, classifications, pipelineFilters, limit, semanticWeight); // relevance first, freshness second: the temporal boost only reorders what fusion already selected const rerankedCandidates = applyTemporalRerank(candidates, { recencyWeight: this.getRecencyWeight(), - halfLifeDays: this.getRecencyHalfLifeDays(), + halfLifeDays: DEFAULT_INTELLIGENT_SEARCH_RECENCY_HALF_LIFE_DAYS, }); return this.normalizeIntelligentResults(rerankedCandidates, userId, limit); diff --git a/apps/meteor/server/settings/ai.ts b/apps/meteor/server/settings/ai.ts index f21fbdef64345..ab1a6819c515a 100644 --- a/apps/meteor/server/settings/ai.ts +++ b/apps/meteor/server/settings/ai.ts @@ -52,23 +52,6 @@ export const createAISettings = async (): Promise => { i18nDescription: 'AI_Intelligent_Search_Enabled_Description', }); - await settingsRegistry.add('AI_Intelligent_Search_Mode', 'semantic', { - group: AI_SETTINGS_GROUP, - section: 'Intelligent_Search', - type: 'select', - values: [ - { key: 'semantic', i18nLabel: 'AI_Intelligent_Search_Mode_Semantic' }, - { key: 'keyword', i18nLabel: 'AI_Intelligent_Search_Mode_Keyword' }, - { key: 'hybrid', i18nLabel: 'AI_Intelligent_Search_Mode_Hybrid' }, - ], - i18nLabel: 'AI_Intelligent_Search_Mode', - i18nDescription: 'AI_Intelligent_Search_Mode_Description', - enterprise: true, - modules: [AI_LICENSE_MODULE], - invalidValue: 'semantic', - enableQuery: { _id: 'AI_Intelligent_Search_Enabled', value: true }, - }); - await settingsRegistry.add('AI_Intelligent_Search_Semantic_Weight', 50, { group: AI_SETTINGS_GROUP, section: 'Intelligent_Search', @@ -78,7 +61,7 @@ export const createAISettings = async (): Promise => { enterprise: true, modules: [AI_LICENSE_MODULE], invalidValue: 50, - enableQuery: { _id: 'AI_Intelligent_Search_Mode', value: 'hybrid' }, + enableQuery: { _id: 'AI_Intelligent_Search_Enabled', value: true }, }); await settingsRegistry.add('AI_Intelligent_Search_Recency_Weight', 0, { @@ -93,18 +76,6 @@ export const createAISettings = async (): Promise => { enableQuery: { _id: 'AI_Intelligent_Search_Enabled', value: true }, }); - await settingsRegistry.add('AI_Intelligent_Search_Recency_Half_Life_Days', 30, { - group: AI_SETTINGS_GROUP, - section: 'Intelligent_Search', - type: 'int', - i18nLabel: 'AI_Intelligent_Search_Recency_Half_Life_Days', - i18nDescription: 'AI_Intelligent_Search_Recency_Half_Life_Days_Description', - enterprise: true, - modules: [AI_LICENSE_MODULE], - invalidValue: 30, - enableQuery: { _id: 'AI_Intelligent_Search_Recency_Weight', value: { $gt: 0 } }, - }); - await settingsRegistry.add('AI_Intelligent_Search_Pipeline_Base_URL', '', { group: AI_SETTINGS_GROUP, section: 'Intelligent_Search', diff --git a/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts b/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts index 69182c378690f..b27bd3a532fbc 100644 --- a/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts +++ b/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts @@ -71,10 +71,8 @@ const cursor = (items: T[]): CursorResult => ({ const settings: Record = { AI_Intelligent_Search_Enabled: true, - AI_Intelligent_Search_Mode: 'semantic', - AI_Intelligent_Search_Semantic_Weight: 50, + AI_Intelligent_Search_Semantic_Weight: 100, AI_Intelligent_Search_Recency_Weight: 0, - AI_Intelligent_Search_Recency_Half_Life_Days: 30, AI_Intelligent_Search_Pipeline_Base_URL: 'https://pipeline.example.com', AI_Intelligent_Search_Pipeline_ID: 'workspace', AI_Intelligent_Search_API_Key: 'key', @@ -202,7 +200,7 @@ describe('AISearchService', () => { const [, options] = serverFetch.firstCall.args; const body = JSON.parse(options.body); // the retriever is asked for a candidate pool, not the requested page - expect(body.params.k).to.equal(50); + expect(body.params.k).to.equal(20); expect(body.filters).to.deep.equal({ room_id: { $in: subscribedRoomIds }, }); @@ -245,17 +243,8 @@ describe('AISearchService', () => { expect(requestBody.params).to.not.have.property('threshold'); }); - it('falls back to keyword path when hybrid semantic weight is 0', async () => { - cachedSettings.get.callsFake((key: string) => { - if (key === 'AI_Intelligent_Search_Semantic_Weight') { - return 0; - } - if (key === 'AI_Intelligent_Search_Mode') { - return 'hybrid'; - } - - return settings[key]; - }); + it('queries only the keyword retriever when the balance is 0', async () => { + cachedSettings.get.callsFake((key: string) => (key === 'AI_Intelligent_Search_Semantic_Weight' ? 0 : settings[key])); serverFetch.resolves({ ok: true, status: 200, @@ -265,24 +254,15 @@ describe('AISearchService', () => { text: async () => '', }); - await createService().search({ query: 'fruit', userId: 'user-id', searchType: 'hybrid' }); + await createService().search({ query: 'fruit', userId: 'user-id' }); expect(serverFetch.callCount).to.equal(1); const requestBody = JSON.parse(serverFetch.firstCall.args[1].body); expect(requestBody.type).to.equal('search'); }); - it('falls back to semantic path when hybrid semantic weight is 100', async () => { - cachedSettings.get.callsFake((key: string) => { - if (key === 'AI_Intelligent_Search_Semantic_Weight') { - return 100; - } - if (key === 'AI_Intelligent_Search_Mode') { - return 'hybrid'; - } - - return settings[key]; - }); + it('queries only the semantic retriever when the balance is 100', async () => { + cachedSettings.get.callsFake((key: string) => (key === 'AI_Intelligent_Search_Semantic_Weight' ? 100 : settings[key])); serverFetch.resolves({ ok: true, status: 200, @@ -292,7 +272,7 @@ describe('AISearchService', () => { text: async () => '', }); - await createService().search({ query: 'fruit', userId: 'user-id', searchType: 'hybrid' }); + await createService().search({ query: 'fruit', userId: 'user-id' }); expect(serverFetch.callCount).to.equal(1); const requestBody = JSON.parse(serverFetch.firstCall.args[1].body); @@ -301,9 +281,7 @@ describe('AISearchService', () => { }); it('uses weighted hybrid with semantic threshold filtering only on semantic branch', async () => { - cachedSettings.get.callsFake((key: string) => - key === 'AI_Intelligent_Search_Mode' || key === 'AI_Intelligent_Search_Semantic_Weight' ? settings[key] : settings[key], - ); + cachedSettings.get.callsFake((key: string) => (key === 'AI_Intelligent_Search_Semantic_Weight' ? 50 : settings[key])); serverFetch.reset(); serverFetch @@ -329,12 +307,7 @@ describe('AISearchService', () => { text: async () => '', }); - const results = await createService().search({ - query: 'fruit', - userId: 'user-id', - searchType: 'hybrid', - limit: 5, - }); + const results = await createService().search({ query: 'fruit', userId: 'user-id', limit: 5 }); expect(serverFetch.callCount).to.equal(2); expect(results).to.deep.equal([ @@ -365,15 +338,15 @@ describe('AISearchService', () => { serverFetch.resolves({ ok: true, status: 200, json: async () => ({ results: [] }), text: async () => '' }); const service = createService(); - await service.search({ query: 'fruit', userId: 'user-id', limit: 20 }); - expect(JSON.parse(serverFetch.lastCall.args[1].body).params.k).to.equal(60); + await service.search({ query: 'fruit', userId: 'user-id', limit: 9 }); + expect(JSON.parse(serverFetch.lastCall.args[1].body).params.k).to.equal(27); await service.search({ query: 'fruit', userId: 'user-id', limit: 50 }); - expect(JSON.parse(serverFetch.lastCall.args[1].body).params.k).to.equal(100); + expect(JSON.parse(serverFetch.lastCall.args[1].body).params.k).to.equal(50); }); it('keeps a full page of hybrid results when fusion candidates are not visible', async () => { - cachedSettings.get.callsFake((key: string) => (key === 'AI_Intelligent_Search_Mode' ? 'hybrid' : settings[key])); + cachedSettings.get.callsFake((key: string) => (key === 'AI_Intelligent_Search_Semantic_Weight' ? 50 : settings[key])); // only the last two candidates resolve to a visible message Messages.findVisibleByIds.callsFake((msgIds: string[]) => cursor( @@ -401,7 +374,7 @@ describe('AISearchService', () => { .onCall(1) .resolves({ ok: true, status: 200, json: async () => ({ results: keywordResults }), text: async () => '' }); - const results = await createService().search({ query: 'fruit', userId: 'user-id', searchType: 'hybrid', limit: 2 }); + const results = await createService().search({ query: 'fruit', userId: 'user-id', limit: 2 }); expect(results.map(({ _id }: { _id: string }) => _id)).to.deep.equal(['visible-b', 'visible-a']); }); @@ -438,17 +411,8 @@ describe('AISearchService', () => { expect(withBoost.map(({ _id }: { _id: string }) => _id)).to.deep.equal(['fresh', 'stale']); }); - it('falls back to a sane half-life when the setting is not usable', async () => { - cachedSettings.get.callsFake((key: string) => { - if (key === 'AI_Intelligent_Search_Recency_Weight') { - return 50; - } - if (key === 'AI_Intelligent_Search_Recency_Half_Life_Days') { - return 0; - } - - return settings[key]; - }); + it('lets an explicit searchType pin an endpoint of the balance without an admin change', async () => { + cachedSettings.get.callsFake((key: string) => (key === 'AI_Intelligent_Search_Semantic_Weight' ? 50 : settings[key])); serverFetch.resolves({ ok: true, status: 200, @@ -456,9 +420,19 @@ describe('AISearchService', () => { text: async () => '', }); - const results = await createService().search({ query: 'fruit', userId: 'user-id', limit: 5 }); + const service = createService(); + await service.search({ query: 'fruit', userId: 'user-id', searchType: 'semantic' }); + expect(serverFetch.callCount).to.equal(1); + expect(JSON.parse(serverFetch.lastCall.args[1].body).type).to.equal('similarity'); + + serverFetch.resetHistory(); + await service.search({ query: 'fruit', userId: 'user-id', searchType: 'keyword' }); + expect(serverFetch.callCount).to.equal(1); + expect(JSON.parse(serverFetch.lastCall.args[1].body).type).to.equal('search'); - expect(results.map(({ _id }: { _id: string }) => _id)).to.deep.equal(['allowed-msg']); + serverFetch.resetHistory(); + await service.search({ query: 'fruit', userId: 'user-id', searchType: 'hybrid' }); + expect(serverFetch.callCount).to.equal(2); }); it('resolves room-name filters before querying the pipeline', async () => { diff --git a/docs/features/ai-search-hybrid-benchmark.md b/docs/features/ai-search-hybrid-benchmark.md index 9180153a849d3..713f2af15e3e4 100644 --- a/docs/features/ai-search-hybrid-benchmark.md +++ b/docs/features/ai-search-hybrid-benchmark.md @@ -4,6 +4,15 @@ Measurements behind the hybrid search defaults. Re-run these before changing `INTELLIGENT_SEARCH_CANDIDATE_MULTIPLIER`, `MIN_INTELLIGENT_SEARCH_CANDIDATES`, `INTELLIGENT_SEARCH_RRF_CONSTANT`, or the shipped setting defaults. +## Latency + +Hybrid issues both retriever requests with `Promise.all`, so its latency is the *slower* branch, not the +sum. At the shipped pool sizes both branches sit in the same 550-700 ms p50 band, so hybrid costs +roughly one retrieval, plus fusion and reranking which are in-memory over at most 100 candidates. + +The extremes of the balance (`0` and `100`) issue **one** request, not two — the unused retriever is +never called. + ## Method - **Corpus**: 547 synthetic Rocket.Chat messages ingested into a QA Intelligent Search pipeline — @@ -41,17 +50,30 @@ Absolute numbers are only meaningful relative to each other: the corpus is synth of a 20-query synthetic set. 50 is the neutral, defensible midpoint; revisit with judged production queries rather than promoting 60 on this evidence. -## Candidate pool sweep (w = 60) +## Candidate pool: the quality/latency frontier + +Quality alone would pick a pool of 50. Latency says otherwise. Pipeline round-trip measured over +20 queries × 3 repetitions: -| candidate pool | best nDCG@10 | conceptual | -| --- | --- | --- | -| 20 | 0.6881 | 0.6014 | -| **50** | **0.7152** | 0.6303 | -| 100 | 0.6987 | 0.5677 | +| pool (k) | semantic nDCG@10 | best hybrid nDCG@10 | semantic p50 | semantic p95 | +| --- | --- | --- | --- | --- | +| 5 (pre-feature default) | 0.6377 | 0.6438 | 588 ms | 820 ms | +| **20** | **0.6819** | **0.6881** | **584 ms** | 709 ms | +| 50 | 0.7091 | 0.7152 | 910 ms | 1342 ms | +| 100 | 0.6925 | 0.6987 | 1490 ms | 2638 ms | -50 per branch is the peak. 20 starves fusion; 100 dilutes conceptual queries with weak neighbours. -Hence `MIN_INTELLIGENT_SEARCH_CANDIDATES = 50` — the default page size of 5 lands exactly on the -optimum, and larger pages scale by ×3 up to the cap of 100. +- **20 is free**: it costs the same as the old pool of 5 (584 ms vs 588 ms p50) and lifts nDCG@10 by + **6.9%**. Below ~20 the pipeline's vector index is clearly not searching hard enough. +- **50 is not free**: +4.0% nDCG for **+56% latency**. Wrong trade for navbar typeahead, which fires on + every debounced keystroke. +- **100 is strictly worse**: slower *and* lower quality than 50. + +Hence `MIN_INTELLIGENT_SEARCH_CANDIDATES = 20`, `MAX_INTELLIGENT_SEARCH_CANDIDATES = 50`, multiplier ×3. +The navbar (`limit` 5) lands on 20 — same latency as before the feature, better relevance. The search +page (`limit` 9, growing to 50 on *Show more*) scales to the 50 cap, where the extra latency is paid by a +deliberate full-page search rather than by typeahead. + +Keyword-branch latency is flat across k (547-666 ms p50), so the pool size is a semantic-side cost. ## Temporal boost sweep (w = 60, candidate pool 50) @@ -73,8 +95,14 @@ the boost never displaces an exact-identifier match. Aggressive settings are actively harmful: weight 100 with a 7-day half-life drops conceptual queries from 0.6303 to 0.4695. Short half-lives are sharp and unforgiving; 30-90 days are stable. -**Shipped default: weight 0 (disabled), half-life 30.** Hybrid relevance ships first and ranking stays -unchanged unless an admin opts in. Recommended starting point when enabling: **weight 25, half-life 30**. +**Shipped default: weight 0 (disabled).** Hybrid relevance ships first and ranking stays unchanged +unless an admin opts in. Recommended starting point when enabling: **weight 25**. + +The half-life is **not** an admin setting - it is fixed at 30 days. Across the useful weight range the +30-day and 90-day columns differ by well under 1% nDCG (0.7507 vs 0.7510 at weight 25), so the knob buys +no reachable quality. Only the 7-day column behaves differently, and it behaves *worse*. If half-life +ever needs to move, change the constant on the evidence of a fresh sweep rather than delegating it to +admins. ## Backend limitations found while benchmarking diff --git a/docs/features/ai-search-hybrid.md b/docs/features/ai-search-hybrid.md index 5b4dff72c67ba..7daf56b5d1300 100644 --- a/docs/features/ai-search-hybrid.md +++ b/docs/features/ai-search-hybrid.md @@ -2,15 +2,28 @@ ## Overview -AI Search retrieves messages from an external Intelligent Search pipeline. It supports three retrieval -modes, selected by the `AI_Intelligent_Search_Mode` setting and overridable per request via the -`searchType` query parameter on `GET /v1/ai.search`: +AI Search retrieves messages from an external Intelligent Search pipeline. Both retrievers are the *same* +pipeline endpoint (`POST /pipelines/{id}/search`), distinguished only by the request body: -| Mode | Pipeline request | Notes | +| Retriever | Pipeline request | Threshold sent | | --- | --- | --- | -| `semantic` (default) | `type: "similarity"`, `search_type: 2` | Vector retrieval. The only mode before this feature. | -| `keyword` | `type: "search"`, `search_type: 1` | Full-text retrieval. | -| `hybrid` | both, in parallel | Fused client-side with weighted RRF. | +| semantic | `type: "similarity"`, `classification.search_type: 2` | yes | +| keyword | `type: "search"`, `classification.search_type: 1` | no | + +Which of them runs is decided by a single setting, `AI_Intelligent_Search_Semantic_Weight` (0-100): + +| Balance | Behaviour | +| --- | --- | +| `0` | keyword only - the semantic retriever is never called | +| `1`-`99` | both in parallel, fused with weighted RRF | +| `100` | semantic only - the keyword retriever is never called | + +There is deliberately **no separate "search mode" setting**: the balance already expresses every mode, +and a second control would only let the two disagree. + +A caller can pin an endpoint of that range per request with the `searchType` query parameter on +`GET /v1/ai.search` (`keyword` maps to 0, `semantic` to 100, `hybrid` to whatever the setting says), +which is useful for evaluation without changing workspace configuration. ## Pipeline retrieval @@ -94,8 +107,10 @@ Applied after fusion, so relevance selects the candidates and freshness only reo final(d) = score(d) × (1 + recencyWeight × 2^(-ageInDays / halfLifeDays)) ``` -- `AI_Intelligent_Search_Recency_Weight` (0-100, default **0** = disabled). -- `AI_Intelligent_Search_Recency_Half_Life_Days` (default 30). +`AI_Intelligent_Search_Recency_Weight` (0-100, default **0** = disabled) is the only control. The +half-life is fixed at 30 days (`DEFAULT_INTELLIGENT_SEARCH_RECENCY_HALF_LIFE_DAYS`): offline sweeps found +the 30-90 day range essentially flat, so exposing it would add a setting without adding reachable +quality. Timestamps come from the pipeline fragment metadata, so the boost costs no extra database work. Candidates without a usable timestamp keep their relevance score rather than being penalised. diff --git a/packages/ai-search/src/constants.ts b/packages/ai-search/src/constants.ts index 1797c49f7b1bf..6446e05bd9415 100644 --- a/packages/ai-search/src/constants.ts +++ b/packages/ai-search/src/constants.ts @@ -6,11 +6,13 @@ export const AI_SEARCH_FILTER_SUGGESTION_LIMIT = 5; export const AI_SEARCH_ROOM_LOOKUP_LIMIT = 20; export const MAX_INTELLIGENT_SEARCH_RESULTS = 50; // Candidate pool retrieved from each retriever before fusion. Implementation detail, never exposed to -// admins. The floor of 50 is where offline nDCG@10 peaked: 20 starves fusion, 100 dilutes conceptual -// queries with weak neighbours. See docs/features/ai-search-hybrid-benchmark.md. +// admins, and tuned on the measured quality/latency frontier rather than on quality alone: +// a pool of 20 costs the same as the old pool of 5 (p50 584ms vs 588ms) but lifts nDCG@10 by ~7%, +// while 100 is both slower (p50 1490ms) and *worse* than 50. Hence floor 20, cap 50. +// See docs/features/ai-search-hybrid-benchmark.md. export const INTELLIGENT_SEARCH_CANDIDATE_MULTIPLIER = 3; -export const MIN_INTELLIGENT_SEARCH_CANDIDATES = 50; -export const MAX_INTELLIGENT_SEARCH_CANDIDATES = 100; +export const MIN_INTELLIGENT_SEARCH_CANDIDATES = 20; +export const MAX_INTELLIGENT_SEARCH_CANDIDATES = 50; export const INTELLIGENT_SEARCH_RRF_CONSTANT = 60; export const DEFAULT_INTELLIGENT_SEARCH_SEMANTIC_WEIGHT = 50; export const DEFAULT_INTELLIGENT_SEARCH_RECENCY_HALF_LIFE_DAYS = 30; diff --git a/packages/i18n/src/locales/en.i18n.json b/packages/i18n/src/locales/en.i18n.json index c3f5f7cd8323f..4875a69d49423 100644 --- a/packages/i18n/src/locales/en.i18n.json +++ b/packages/i18n/src/locales/en.i18n.json @@ -582,23 +582,16 @@ "AI_Intelligent_Search_Enabled_Description": "Use the configured vector-search pipeline to add semantic results to workspace search.", "AI_Intelligent_Search_Min_Similarity_Percent": "Minimum semantic similarity (%)", "AI_Intelligent_Search_Min_Similarity_Percent_Description": "Higher values return fewer but closer semantic matches. Use 0 to keep the widest result set.", - "AI_Intelligent_Search_Mode": "Search method", - "AI_Intelligent_Search_Mode_Description": "How AI Search retrieves messages. Hybrid combines keyword and semantic retrieval.", - "AI_Intelligent_Search_Mode_Hybrid": "Hybrid", - "AI_Intelligent_Search_Mode_Keyword": "Keyword only", - "AI_Intelligent_Search_Mode_Semantic": "Semantic only", "AI_Intelligent_Search_Pipeline_Base_URL": "Pipeline API base URL", "AI_Intelligent_Search_Pipeline_Base_URL_Description": "Base URL for the intelligent-search pipeline API.", "AI_Intelligent_Search_Pipeline_ID": "Pipeline ID", "AI_Intelligent_Search_Pipeline_ID_Description": "Identifier of the target pipeline used for semantic search requests.", "AI_Intelligent_Search_Query_Template": "Query template", "AI_Intelligent_Search_Query_Template_Description": "Optional template applied to search queries before sending to the pipeline. Use {query} as the placeholder. Leave blank to send the raw query.", - "AI_Intelligent_Search_Recency_Half_Life_Days": "Recency half-life (days)", - "AI_Intelligent_Search_Recency_Half_Life_Days_Description": "Age at which a message keeps half of its recency boost.", "AI_Intelligent_Search_Recency_Weight": "Recency boost", - "AI_Intelligent_Search_Recency_Weight_Description": "How strongly newer messages are promoted after relevance ranking. Use 0 to rank purely by relevance.", - "AI_Intelligent_Search_Semantic_Weight": "Hybrid search balance", - "AI_Intelligent_Search_Semantic_Weight_Description": "Relative influence of semantic retrieval in hybrid mode: 0 is keyword only, 100 is semantic only. This is not the percentage of results that come from semantic search.", + "AI_Intelligent_Search_Recency_Weight_Description": "How strongly newer messages are promoted after relevance ranking. A message keeps half of its boost every 30 days. Use 0 to rank purely by relevance.", + "AI_Intelligent_Search_Semantic_Weight": "Search balance", + "AI_Intelligent_Search_Semantic_Weight_Description": "Balance between keyword and semantic retrieval: 0 searches by keyword only, 100 by meaning only, and anything in between combines both. This is not the percentage of results that come from semantic search.", "AI_LLM_OpenAI_API_Key": "API key", "AI_LLM_OpenAI_API_Key_Description": "API key for the OpenAI-compatible chat completions endpoint.", "AI_LLM_OpenAI_Base_URL": "API base URL", From c70dd2d9953b94af2a120705afbed800e4d6ca1d Mon Sep 17 00:00:00 2001 From: Dnouv Date: Fri, 11 Sep 2026 22:57:49 +0800 Subject: [PATCH 03/10] fix: do not report a full-text rank as a semantic similarity Both retrievers report their number in the same `score` field, but they mean opposite things: the semantic branch returns a cosine distance where lower is better, the keyword branch a full-text rank where higher is better. normalizeIntelligentSearchCandidates read both as a distance, so every keyword hit surfaced a fabricated similarity, inverted: a strong lexical match with rank 0.2803 displayed as 72% while a weak one at 0.0183 displayed as 98%. That value reaches the results UI and is passed to answer generation as a relevance signal. Keyword candidates now carry `keywordScore` for observability and no `score`, so a match percentage is shown only where one genuinely exists. Fusion is unaffected: it ranks, and never read these values. --- .../services/ai-search/service.tests.ts | 2 +- docs/features/ai-search-hybrid.md | 10 ++++- .../ai-search/src/intelligentSearch.spec.ts | 44 +++++++++++++++++-- packages/ai-search/src/intelligentSearch.ts | 12 ++++- packages/ai-search/src/types.ts | 3 ++ 5 files changed, 64 insertions(+), 7 deletions(-) diff --git a/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts b/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts index b27bd3a532fbc..f6fe2b794d126 100644 --- a/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts +++ b/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts @@ -322,13 +322,13 @@ describe('AISearchService', () => { room: { _id: 'allowed', t: 'c', name: 'general', fname: 'General' }, }, { + // keyword-sourced: carries no similarity, because the pipeline's full-text rank is not one _id: 'keyword-msg', rid: 'allowed', msgId: 'keyword-msg', text: 'keyword-msg from db', ts: '2026-01-05T12:00:00.000Z', u: { username: 'alice', name: 'Alice' }, - score: 0.6, room: { _id: 'allowed', t: 'c', name: 'general', fname: 'General' }, }, ]); diff --git a/docs/features/ai-search-hybrid.md b/docs/features/ai-search-hybrid.md index 7daf56b5d1300..703db930a492b 100644 --- a/docs/features/ai-search-hybrid.md +++ b/docs/features/ai-search-hybrid.md @@ -72,8 +72,14 @@ The two retrievers report scores on incompatible scales, verified empirically ag - Full-text `score` is a **rank**: *higher is better*. `normalizeIntelligentSearchCandidates` converts distance to similarity (`similarity = 1 - distance`) for -display and for the similarity guardrail. Fusion deliberately never compares the two raw scores — it -works on **rank positions only**, which is what makes the incompatible scales a non-problem. +display and for the similarity guardrail, and does so **only for semantic candidates**. Both retrievers +report their number in the same `score` field, so reading a keyword hit's rank as a distance would invert +it and fabricate a confident similarity — the best lexical hit would display the lowest score. Keyword +candidates therefore carry `keywordScore` for observability and no `score` at all; the UI shows a match +percentage only where one genuinely exists. + +Fusion deliberately never compares the two raw scores — it works on **rank positions only**, which is +what makes the incompatible scales a non-problem. ## Weighted RRF diff --git a/packages/ai-search/src/intelligentSearch.spec.ts b/packages/ai-search/src/intelligentSearch.spec.ts index 627ee1d2d5a2b..cbd553be82260 100644 --- a/packages/ai-search/src/intelligentSearch.spec.ts +++ b/packages/ai-search/src/intelligentSearch.spec.ts @@ -114,15 +114,53 @@ describe('AI Search intelligent search helpers', () => { rid: 'r1', msgId: 'm1', pipelineText: 'keyword match', - score: 0.58, - semanticSimilarity: 0.58, - semanticDistance: 0.42, + keywordScore: 0.42, source: 'keyword', }, ]); }); }); + describe('keyword candidate scores', () => { + it('never reports a full-text rank as a semantic similarity', () => { + // the pipeline reuses `score` for a full-text rank where higher is better, so reading it as a + // cosine distance would both invert the ordering and fabricate a confident similarity + const [best, worst] = normalizeIntelligentSearchCandidates( + { + results: [ + { metadata: { room_id: 'r1', msg_id: 'm1' }, score: 0.2803 }, + { metadata: { room_id: 'r2', msg_id: 'm2' }, score: 0.0183 }, + ], + }, + [], + 10, + undefined, + 'keyword', + ); + + expect(best).not.toHaveProperty('score'); + expect(best).not.toHaveProperty('semanticSimilarity'); + expect(best).not.toHaveProperty('semanticDistance'); + expect(best.keywordScore).toBe(0.2803); + expect(worst.keywordScore).toBe(0.0183); + expect(best.source).toBe('keyword'); + }); + + it('still reports semantic similarity for semantic candidates', () => { + const [candidate] = normalizeIntelligentSearchCandidates( + { results: [{ metadata: { room_id: 'r1', msg_id: 'm1' }, score: 0.2 }] }, + [], + 10, + undefined, + 'semantic', + ); + + expect(candidate.score).toBe(0.8); + expect(candidate.semanticSimilarity).toBe(0.8); + expect(candidate).not.toHaveProperty('keywordScore'); + }); + }); + describe('candidate timestamps', () => { it('carries the pipeline timestamp through for the temporal rerank stage', () => { const [withMetadataTs, withResultTs, withoutTs] = normalizeIntelligentSearchCandidates( diff --git a/packages/ai-search/src/intelligentSearch.ts b/packages/ai-search/src/intelligentSearch.ts index a8956f93361fb..6d37de54f516f 100644 --- a/packages/ai-search/src/intelligentSearch.ts +++ b/packages/ai-search/src/intelligentSearch.ts @@ -66,10 +66,18 @@ const normalizePipelineScore = (value: number): number => { return Math.min(1, Math.max(0, normalizedValue)); }; +// Only the semantic retriever reports a cosine distance. The keyword retriever reuses the same `score` +// field for a full-text rank, where *higher* is better, so interpreting it as a distance would invert it +// and surface a confident-looking similarity for a weak lexical match. const extractPipelineSimilarityScores = ( result: Record, metadata: Record, + source: IntelligentSearchCandidateSource, ): { semanticSimilarity?: number; semanticDistance?: number } => { + if (source === 'keyword') { + return {}; + } + const similarity = firstNumber(result.similarity, metadata.similarity); if (typeof similarity === 'number') { const semanticSimilarity = normalizePipelineScore(similarity); @@ -159,7 +167,8 @@ export const normalizeIntelligentSearchCandidates = ( continue; } - const { semanticDistance, semanticSimilarity } = extractPipelineSimilarityScores(result, metadata); + const { semanticDistance, semanticSimilarity } = extractPipelineSimilarityScores(result, metadata, source); + const keywordScore = source === 'keyword' ? firstNumber(result.score, metadata.score) : undefined; const ts = firstString(metadata.timestamp, result.timestamp); candidates.push({ _id: msgId || `intelligent-${index}`, @@ -172,6 +181,7 @@ export const normalizeIntelligentSearchCandidates = ( semanticSimilarity, semanticDistance, }), + ...(typeof keywordScore === 'number' && { keywordScore }), ...(source && { source }), }); } diff --git a/packages/ai-search/src/types.ts b/packages/ai-search/src/types.ts index c4943cd02442e..0e73a9555f810 100644 --- a/packages/ai-search/src/types.ts +++ b/packages/ai-search/src/types.ts @@ -59,9 +59,12 @@ export type IntelligentSearchCandidate = { rid?: string; msgId?: string; pipelineText: string; + /** normalized cosine similarity; only ever set for semantic candidates */ score?: number; semanticSimilarity?: number; semanticDistance?: number; + /** raw full-text rank reported by the pipeline, kept for observability and never shown as a similarity */ + keywordScore?: number; source?: IntelligentSearchCandidateSource; /** message timestamp reported by the pipeline, used by the temporal rerank stage */ ts?: string; From 6d0b83f32502c1d039de4ea5d2dc7fd4612f54c4 Mon Sep 17 00:00:00 2001 From: Dnouv Date: Fri, 11 Sep 2026 23:03:42 +0800 Subject: [PATCH 04/10] fix: keep hybrid search alive when one retriever fails Review follow-ups on the hybrid retrieval work. The two branches were issued with Promise.all, but searchIntelligentPipeline rethrows on network failure and on the 10s timeout (it only swallows non-2xx). So a single flaky keyword request rejected the pair, search() propagated, and the endpoint returned zero results while a perfectly good semantic result set was discarded. They now go through Promise.allSettled and degrade to whichever retriever survived; only a double failure propagates. Also branch-qualifies the synthetic candidate id. normalizeIntelligentSearchCandidates falls back to `intelligent-${index}` when a result has no msgId, and the index is per-retriever, so semantic result #0 and keyword result #0 fused into a single entry with a summed score as though both retrievers had agreed on it. Drops the keywordScore field added in the previous commit: nothing read it. The point of that change was to stop fabricating a similarity for keyword hits, and that stands on its own - such hits now carry no score, and the results UI shows a match percentage only where one honestly exists. --- .../server/services/ai-search/service.ts | 36 ++++++++++++++----- .../services/ai-search/service.tests.ts | 32 +++++++++++++++++ docs/features/ai-search-hybrid.md | 9 +++-- packages/ai-search/src/fusion.spec.ts | 12 +++++++ .../ai-search/src/intelligentSearch.spec.ts | 5 +-- packages/ai-search/src/intelligentSearch.ts | 6 ++-- packages/ai-search/src/types.ts | 8 +++-- 7 files changed, 88 insertions(+), 20 deletions(-) diff --git a/apps/meteor/server/services/ai-search/service.ts b/apps/meteor/server/services/ai-search/service.ts index b7e831ec578a9..cbf8bdc0392fb 100644 --- a/apps/meteor/server/services/ai-search/service.ts +++ b/apps/meteor/server/services/ai-search/service.ts @@ -169,7 +169,9 @@ export class AISearchService extends ServiceClass implements IAISearchService { /** * The 0-100 balance is the whole retrieval control: 0 is keyword only, 100 is semantic only, anything - * between fuses both. A per-request `searchType` pins an endpoint of that range without an admin change. + * between fuses both. A per-request `searchType` pins an endpoint of that range without an admin + * change; `hybrid` deliberately defers to the configured balance rather than forcing a mid value, so + * a workspace pinned to one retriever stays pinned. */ private resolveSemanticWeight(searchType: IntelligentSearchType | undefined): number { if (searchType === 'keyword') { @@ -249,14 +251,32 @@ export class AISearchService extends ServiceClass implements IAISearchService { return toRankedCandidates(filterSemanticCandidatesByMinimumSimilarity(await queryBranch('semantic'), minimumSimilarityPercent)); } - const [semanticCandidates, keywordCandidates] = await Promise.all([queryBranch('semantic'), queryBranch('keyword')]); + // a retriever that throws must not take the other one down with it: a flaky keyword branch + // should degrade hybrid to semantic-only results rather than to an empty result set + const [semanticResult, keywordResult] = await Promise.allSettled([queryBranch('semantic'), queryBranch('keyword')]); + const keywordCandidates = keywordResult.status === 'fulfilled' ? keywordResult.value : undefined; + const semanticCandidates = + semanticResult.status === 'fulfilled' + ? filterSemanticCandidatesByMinimumSimilarity(semanticResult.value, minimumSimilarityPercent) + : undefined; + + if (!semanticCandidates) { + if (!keywordCandidates) { + throw semanticResult.status === 'rejected' ? semanticResult.reason : new Error('error-ai-search-retrieval-failed'); + } - return fuseCandidatesWithWeightedRRF( - filterSemanticCandidatesByMinimumSimilarity(semanticCandidates, minimumSimilarityPercent), - keywordCandidates, - semanticWeight, - candidateLimit, - ); + logger.warn({ msg: 'Intelligent search branch failed, serving the surviving retriever', failedBranch: 'semantic' }); + + return toRankedCandidates(keywordCandidates); + } + + if (!keywordCandidates) { + logger.warn({ msg: 'Intelligent search branch failed, serving the surviving retriever', failedBranch: 'keyword' }); + + return toRankedCandidates(semanticCandidates); + } + + return fuseCandidatesWithWeightedRRF(semanticCandidates, keywordCandidates, semanticWeight, candidateLimit); } /** diff --git a/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts b/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts index f6fe2b794d126..1dbab4ffddf87 100644 --- a/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts +++ b/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts @@ -411,6 +411,38 @@ describe('AISearchService', () => { expect(withBoost.map(({ _id }: { _id: string }) => _id)).to.deep.equal(['fresh', 'stale']); }); + it('serves the surviving retriever when one hybrid branch fails', async () => { + cachedSettings.get.callsFake((key: string) => (key === 'AI_Intelligent_Search_Semantic_Weight' ? 50 : settings[key])); + serverFetch.reset(); + serverFetch + .onCall(0) + .resolves({ + ok: true, + status: 200, + json: async () => ({ results: [{ metadata: { room_id: 'allowed', msg_id: 'allowed-msg' }, score: 0.2 }] }), + text: async () => '', + }) + .onCall(1) + .rejects(new Error('keyword branch timed out')); + + const results = await createService().search({ query: 'fruit', userId: 'user-id', limit: 5 }); + + expect(results.map(({ _id }: { _id: string }) => _id)).to.deep.equal(['allowed-msg']); + }); + + it('gives up only when both hybrid branches fail', async () => { + cachedSettings.get.callsFake((key: string) => (key === 'AI_Intelligent_Search_Semantic_Weight' ? 50 : settings[key])); + serverFetch.reset(); + serverFetch.rejects(new Error('pipeline unreachable')); + + await createService() + .search({ query: 'fruit', userId: 'user-id', limit: 5 }) + .then( + () => expect.fail('expected the search to reject'), + (error: Error) => expect(error.message).to.equal('pipeline unreachable'), + ); + }); + it('lets an explicit searchType pin an endpoint of the balance without an admin change', async () => { cachedSettings.get.callsFake((key: string) => (key === 'AI_Intelligent_Search_Semantic_Weight' ? 50 : settings[key])); serverFetch.resolves({ diff --git a/docs/features/ai-search-hybrid.md b/docs/features/ai-search-hybrid.md index 703db930a492b..3ae7e40914a1a 100644 --- a/docs/features/ai-search-hybrid.md +++ b/docs/features/ai-search-hybrid.md @@ -75,8 +75,9 @@ The two retrievers report scores on incompatible scales, verified empirically ag display and for the similarity guardrail, and does so **only for semantic candidates**. Both retrievers report their number in the same `score` field, so reading a keyword hit's rank as a distance would invert it and fabricate a confident similarity — the best lexical hit would display the lowest score. Keyword -candidates therefore carry `keywordScore` for observability and no `score` at all; the UI shows a match -percentage only where one genuinely exists. +candidates therefore carry no `score` at all, and the results UI renders a match percentage only where one +genuinely exists. In a fused list that means the badge appears on semantically-found hits and is absent on +keyword-only hits — an honest gap rather than a fabricated number. Fusion deliberately never compares the two raw scores — it works on **rank positions only**, which is what makes the incompatible scales a non-problem. @@ -92,6 +93,10 @@ score(d) = (1 - w) / (C + rank_fulltext(d)) + w / (C + rank_semantic(d)) A branch that did not return `d` contributes nothing. `w = 0` and `w = 100` short-circuit to a single retriever, so the other branch is never even requested. +The two branches are issued with `Promise.allSettled`, not `Promise.all`: if one retriever throws +(network failure, or the pipeline's 10s timeout) the search degrades to the surviving retriever rather +than returning nothing. Only a double failure propagates. + `C` and the candidate pool size are implementation parameters and are intentionally **not** admin settings. diff --git a/packages/ai-search/src/fusion.spec.ts b/packages/ai-search/src/fusion.spec.ts index bae5c079992fe..9d756a91e92f8 100644 --- a/packages/ai-search/src/fusion.spec.ts +++ b/packages/ai-search/src/fusion.spec.ts @@ -99,6 +99,18 @@ describe('AI Search fusion helpers', () => { expect(ids(fuseCandidatesWithWeightedRRF(semantic, keyword, -20, 2))).toEqual(['shared', 'k1']); }); + it('does not fuse branch-local synthetic ids into a single candidate', () => { + // normalizeIntelligentSearchCandidates qualifies its fallback ids by source precisely so that + // these two unrelated messages cannot be mistaken for one agreed-upon hit + const semanticOnly = { _id: 'intelligent-semantic-0', rid: 'r1', pipelineText: 'a' }; + const keywordOnly = { _id: 'intelligent-keyword-0', rid: 'r2', pipelineText: 'b' }; + + const fused = fuseCandidatesWithWeightedRRF([semanticOnly], [keywordOnly], 50, 10); + + expect(fused).toHaveLength(2); + expect(fused.every(({ rrfScore }) => rrfScore === 0.5 / (INTELLIGENT_SEARCH_RRF_CONSTANT + 1))).toBe(true); + }); + it('ignores candidates without any usable identifier', () => { const unidentified = { _id: '', msgId: '', rid: 'room', pipelineText: '' }; diff --git a/packages/ai-search/src/intelligentSearch.spec.ts b/packages/ai-search/src/intelligentSearch.spec.ts index cbd553be82260..ab276e308b5c8 100644 --- a/packages/ai-search/src/intelligentSearch.spec.ts +++ b/packages/ai-search/src/intelligentSearch.spec.ts @@ -114,7 +114,6 @@ describe('AI Search intelligent search helpers', () => { rid: 'r1', msgId: 'm1', pipelineText: 'keyword match', - keywordScore: 0.42, source: 'keyword', }, ]); @@ -141,8 +140,7 @@ describe('AI Search intelligent search helpers', () => { expect(best).not.toHaveProperty('score'); expect(best).not.toHaveProperty('semanticSimilarity'); expect(best).not.toHaveProperty('semanticDistance'); - expect(best.keywordScore).toBe(0.2803); - expect(worst.keywordScore).toBe(0.0183); + expect(worst).not.toHaveProperty('score'); expect(best.source).toBe('keyword'); }); @@ -157,7 +155,6 @@ describe('AI Search intelligent search helpers', () => { expect(candidate.score).toBe(0.8); expect(candidate.semanticSimilarity).toBe(0.8); - expect(candidate).not.toHaveProperty('keywordScore'); }); }); diff --git a/packages/ai-search/src/intelligentSearch.ts b/packages/ai-search/src/intelligentSearch.ts index 6d37de54f516f..0053bdfbd66b7 100644 --- a/packages/ai-search/src/intelligentSearch.ts +++ b/packages/ai-search/src/intelligentSearch.ts @@ -168,10 +168,11 @@ export const normalizeIntelligentSearchCandidates = ( } const { semanticDistance, semanticSimilarity } = extractPipelineSimilarityScores(result, metadata, source); - const keywordScore = source === 'keyword' ? firstNumber(result.score, metadata.score) : undefined; const ts = firstString(metadata.timestamp, result.timestamp); candidates.push({ - _id: msgId || `intelligent-${index}`, + // branch-qualified: the index is per-retriever, so an unqualified fallback would make + // semantic result #0 and keyword result #0 fuse as if they were the same message + _id: msgId || `intelligent-${source}-${index}`, rid, msgId, pipelineText: firstString(result.text, result.content, result.document, result.page_content, metadata.text) || '', @@ -181,7 +182,6 @@ export const normalizeIntelligentSearchCandidates = ( semanticSimilarity, semanticDistance, }), - ...(typeof keywordScore === 'number' && { keywordScore }), ...(source && { source }), }); } diff --git a/packages/ai-search/src/types.ts b/packages/ai-search/src/types.ts index 0e73a9555f810..8882de9626a46 100644 --- a/packages/ai-search/src/types.ts +++ b/packages/ai-search/src/types.ts @@ -59,12 +59,14 @@ export type IntelligentSearchCandidate = { rid?: string; msgId?: string; pipelineText: string; - /** normalized cosine similarity; only ever set for semantic candidates */ + /** + * Normalized cosine similarity, set only for semantic candidates. Keyword candidates deliberately + * carry no score: the pipeline reports their full-text rank in the same field, and it is not a + * similarity, so there is no honest value to show. + */ score?: number; semanticSimilarity?: number; semanticDistance?: number; - /** raw full-text rank reported by the pipeline, kept for observability and never shown as a similarity */ - keywordScore?: number; source?: IntelligentSearchCandidateSource; /** message timestamp reported by the pipeline, used by the temporal rerank stage */ ts?: string; From 285325b253328b709c5ca76462dedb2643edcc9e Mon Sep 17 00:00:00 2001 From: Dnouv Date: Fri, 11 Sep 2026 23:20:05 +0800 Subject: [PATCH 05/10] chore: trim non-essential comments from AI Search hybrid code --- .../server/services/ai-search/service.ts | 25 ++++++--------- packages/ai-search/src/constants.ts | 7 ++--- packages/ai-search/src/fusion.spec.ts | 2 -- packages/ai-search/src/fusion.ts | 31 ++++++------------- .../ai-search/src/intelligentSearch.spec.ts | 3 +- packages/ai-search/src/intelligentSearch.ts | 14 +++------ packages/ai-search/src/types.ts | 9 ++---- 7 files changed, 28 insertions(+), 63 deletions(-) diff --git a/apps/meteor/server/services/ai-search/service.ts b/apps/meteor/server/services/ai-search/service.ts index cbf8bdc0392fb..7b652fe83c4d9 100644 --- a/apps/meteor/server/services/ai-search/service.ts +++ b/apps/meteor/server/services/ai-search/service.ts @@ -167,12 +167,8 @@ export class AISearchService extends ServiceClass implements IAISearchService { }; } - /** - * The 0-100 balance is the whole retrieval control: 0 is keyword only, 100 is semantic only, anything - * between fuses both. A per-request `searchType` pins an endpoint of that range without an admin - * change; `hybrid` deliberately defers to the configured balance rather than forcing a mid value, so - * a workspace pinned to one retriever stays pinned. - */ + // `hybrid` defers to the configured balance rather than forcing a mid value, so a workspace pinned to + // one retriever stays pinned. private resolveSemanticWeight(searchType: IntelligentSearchType | undefined): number { if (searchType === 'keyword') { return 0; @@ -242,7 +238,7 @@ export class AISearchService extends ServiceClass implements IAISearchService { const minimumSimilarityPercent = Number(config.minimumSimilarityPercent || 0); - // at the extremes only one retriever is worth paying for, so the other is never requested + // at the extremes the other retriever is never requested if (semanticWeight === 0) { return toRankedCandidates(await queryBranch('keyword')); } @@ -251,8 +247,8 @@ export class AISearchService extends ServiceClass implements IAISearchService { return toRankedCandidates(filterSemanticCandidatesByMinimumSimilarity(await queryBranch('semantic'), minimumSimilarityPercent)); } - // a retriever that throws must not take the other one down with it: a flaky keyword branch - // should degrade hybrid to semantic-only results rather than to an empty result set + // allSettled, not all: searchIntelligentPipeline rethrows on network failure and timeout, and one + // flaky branch must degrade hybrid to the survivor rather than to an empty result set const [semanticResult, keywordResult] = await Promise.allSettled([queryBranch('semantic'), queryBranch('keyword')]); const keywordCandidates = keywordResult.status === 'fulfilled' ? keywordResult.value : undefined; const semanticCandidates = @@ -279,10 +275,8 @@ export class AISearchService extends ServiceClass implements IAISearchService { return fuseCandidatesWithWeightedRRF(semanticCandidates, keywordCandidates, semanticWeight, candidateLimit); } - /** - * Retrieval depth per branch. Deliberately larger than the requested page so that fusion has something - * to fuse and so that permission filtering does not eat into the page. Not admin configurable. - */ + // Larger than the requested page so fusion has overlap to work with and permission filtering below + // cannot eat into the page. private getSearchCandidateLimit(requestedLimit: number): number { const scaledLimit = Math.max(requestedLimit, AI_SEARCH_PAGE_SIZE) * INTELLIGENT_SEARCH_CANDIDATE_MULTIPLIER; @@ -356,8 +350,8 @@ export class AISearchService extends ServiceClass implements IAISearchService { userId: string, limit = AI_SEARCH_PAGE_SIZE, ): Promise { - // the whole candidate pool is resolved, not just the first page: permission filtering below can - // drop any candidate, and pre-slicing here would silently return a short page + // the whole pool is resolved, not just the first page: pre-slicing here returns short pages once + // permission filtering below drops a candidate const msgIdSet = new Set(); for (const { msgId } of searchCandidates) { if (msgId) { @@ -505,7 +499,6 @@ export class AISearchService extends ServiceClass implements IAISearchService { const semanticWeight = this.resolveSemanticWeight(searchType); const candidates = await this.buildSearchCandidatesForMode(query, config, classifications, pipelineFilters, limit, semanticWeight); - // relevance first, freshness second: the temporal boost only reorders what fusion already selected const rerankedCandidates = applyTemporalRerank(candidates, { recencyWeight: this.getRecencyWeight(), halfLifeDays: DEFAULT_INTELLIGENT_SEARCH_RECENCY_HALF_LIFE_DAYS, diff --git a/packages/ai-search/src/constants.ts b/packages/ai-search/src/constants.ts index 6446e05bd9415..66fde9e28c4bb 100644 --- a/packages/ai-search/src/constants.ts +++ b/packages/ai-search/src/constants.ts @@ -5,11 +5,8 @@ export const AI_SEARCH_RESULTS_PAGE_SIZE = 8; export const AI_SEARCH_FILTER_SUGGESTION_LIMIT = 5; export const AI_SEARCH_ROOM_LOOKUP_LIMIT = 20; export const MAX_INTELLIGENT_SEARCH_RESULTS = 50; -// Candidate pool retrieved from each retriever before fusion. Implementation detail, never exposed to -// admins, and tuned on the measured quality/latency frontier rather than on quality alone: -// a pool of 20 costs the same as the old pool of 5 (p50 584ms vs 588ms) but lifts nDCG@10 by ~7%, -// while 100 is both slower (p50 1490ms) and *worse* than 50. Hence floor 20, cap 50. -// See docs/features/ai-search-hybrid-benchmark.md. +// Per-retriever candidate pool. Internal, and sized on a measured quality/latency frontier rather than +// quality alone - see docs/features/ai-search-hybrid-benchmark.md before changing these. export const INTELLIGENT_SEARCH_CANDIDATE_MULTIPLIER = 3; export const MIN_INTELLIGENT_SEARCH_CANDIDATES = 20; export const MAX_INTELLIGENT_SEARCH_CANDIDATES = 50; diff --git a/packages/ai-search/src/fusion.spec.ts b/packages/ai-search/src/fusion.spec.ts index 9d756a91e92f8..8e976add03fe7 100644 --- a/packages/ai-search/src/fusion.spec.ts +++ b/packages/ai-search/src/fusion.spec.ts @@ -100,8 +100,6 @@ describe('AI Search fusion helpers', () => { }); it('does not fuse branch-local synthetic ids into a single candidate', () => { - // normalizeIntelligentSearchCandidates qualifies its fallback ids by source precisely so that - // these two unrelated messages cannot be mistaken for one agreed-upon hit const semanticOnly = { _id: 'intelligent-semantic-0', rid: 'r1', pipelineText: 'a' }; const keywordOnly = { _id: 'intelligent-keyword-0', rid: 'r2', pipelineText: 'b' }; diff --git a/packages/ai-search/src/fusion.ts b/packages/ai-search/src/fusion.ts index b7eaa03a65f17..e7431a765e564 100644 --- a/packages/ai-search/src/fusion.ts +++ b/packages/ai-search/src/fusion.ts @@ -17,11 +17,8 @@ const clampPercent = (value: unknown): number => { const getCandidateId = (candidate: IntelligentSearchCandidate): string => candidate.msgId || candidate._id; -/** - * Drops semantic candidates whose similarity is below the configured guardrail. Keyword candidates are - * never scored by the embedding model, so the guardrail deliberately does not apply to them: an exact - * error code or ticket id must not be discarded because it is semantically unremarkable. - */ +// Deliberately does not apply to keyword candidates: an exact error code or ticket id must not be +// dropped for being semantically unremarkable. export const filterSemanticCandidatesByMinimumSimilarity = ( candidates: IntelligentSearchCandidate[], minimumSimilarityPercent: number, @@ -35,13 +32,8 @@ export const filterSemanticCandidatesByMinimumSimilarity = ( return candidates.filter((candidate) => candidate.semanticSimilarity === undefined || candidate.semanticSimilarity >= threshold); }; -/** - * Weighted Reciprocal Rank Fusion. - * - * The two retrievers report scores on incompatible scales (the pipeline returns cosine *distance* for - * semantic hits and a full-text rank for keyword hits), so fusion works on ranks only and the raw scores - * never meet. `semanticWeight` is the admin-facing 0-100 balance: 0 is keyword-only, 100 semantic-only. - */ +// Fuses on rank only: the retrievers report cosine distance and full-text rank respectively, so their +// raw scores are not comparable and must never meet. `semanticWeight` is the 0-100 admin balance. export const fuseCandidatesWithWeightedRRF = ( semanticCandidates: IntelligentSearchCandidate[], keywordCandidates: IntelligentSearchCandidate[], @@ -74,7 +66,6 @@ export const fuseCandidatesWithWeightedRRF = ( } fused.set(candidateId, { - // a candidate found by both retrievers keeps the semantic similarity for display ...existing, ...(branch === 'semantic' && { score: candidate.score ?? existing.score, @@ -107,10 +98,8 @@ export const fuseCandidatesWithWeightedRRF = ( .slice(0, limit); }; -/** - * Turns an already relevance-ordered list into fused candidates so that every retrieval mode reaches the - * temporal stage with a comparable rank-based score. - */ +// Gives single-retriever results the same rank-based score as fusion, so every mode reaches the +// temporal stage comparably scored. export const toRankedCandidates = ( candidates: IntelligentSearchCandidate[], rrfConstant: number = INTELLIGENT_SEARCH_RRF_CONSTANT, @@ -129,11 +118,9 @@ export const getRecencyDecay = (ageInDays: number, halfLifeDays: number): number return 2 ** (-Math.max(0, ageInDays) / halfLifeDays); }; -/** - * Multiplicative temporal boost applied after relevance fusion. A candidate posted right now scores - * `1 + recencyWeight` times its relevance, decaying by half every `halfLifeDays`. Candidates without a - * usable timestamp are left untouched rather than penalised, so a missing `ts` can never demote a hit. - */ +// Multiplicative and bounded by `1 + recencyWeight`, so freshness reorders near-ties but cannot +// overturn a real relevance gap. A missing or unparseable `ts` keeps the relevance score rather than +// being penalised. export const applyTemporalRerank = ( candidates: FusedIntelligentSearchCandidate[], { recencyWeight, halfLifeDays, now = new Date() }: TemporalRerankOptions, diff --git a/packages/ai-search/src/intelligentSearch.spec.ts b/packages/ai-search/src/intelligentSearch.spec.ts index ab276e308b5c8..4a91e2ff48e64 100644 --- a/packages/ai-search/src/intelligentSearch.spec.ts +++ b/packages/ai-search/src/intelligentSearch.spec.ts @@ -122,8 +122,7 @@ describe('AI Search intelligent search helpers', () => { describe('keyword candidate scores', () => { it('never reports a full-text rank as a semantic similarity', () => { - // the pipeline reuses `score` for a full-text rank where higher is better, so reading it as a - // cosine distance would both invert the ordering and fabricate a confident similarity + // 0.2803 is the stronger lexical hit; read as a distance it would display as the weaker one const [best, worst] = normalizeIntelligentSearchCandidates( { results: [ diff --git a/packages/ai-search/src/intelligentSearch.ts b/packages/ai-search/src/intelligentSearch.ts index 0053bdfbd66b7..f668840e83dbe 100644 --- a/packages/ai-search/src/intelligentSearch.ts +++ b/packages/ai-search/src/intelligentSearch.ts @@ -56,19 +56,16 @@ export const normalizeSimilarityPercent = (value: unknown): number => { export const getSemanticDistanceThreshold = (minimumSimilarityPercent: number): number => Number((1 - minimumSimilarityPercent / 100).toFixed(4)); -// pipeline contract (verified against the Intelligent Search API): -// - `score`/`distance` are cosine *distances* - lower is better, in [0,1] -// - `similarity` values are cosine *similarities* - higher is better, in [0,1] -// Both are clamped to [0,1]; percentages are accepted for resilience against provider drift. +// Pipeline contract, verified against a live pipeline: `score`/`distance` are cosine distances (lower is +// better), `similarity` values are cosine similarities. Percentages are accepted for provider drift. const normalizePipelineScore = (value: number): number => { const normalizedValue = Math.abs(value) > 1 ? value / 100 : value; return Math.min(1, Math.max(0, normalizedValue)); }; -// Only the semantic retriever reports a cosine distance. The keyword retriever reuses the same `score` -// field for a full-text rank, where *higher* is better, so interpreting it as a distance would invert it -// and surface a confident-looking similarity for a weak lexical match. +// The keyword retriever reuses `score` for a full-text rank where higher is better, so reading it as a +// distance would invert it and fabricate a confident similarity. const extractPipelineSimilarityScores = ( result: Record, metadata: Record, @@ -170,8 +167,7 @@ export const normalizeIntelligentSearchCandidates = ( const { semanticDistance, semanticSimilarity } = extractPipelineSimilarityScores(result, metadata, source); const ts = firstString(metadata.timestamp, result.timestamp); candidates.push({ - // branch-qualified: the index is per-retriever, so an unqualified fallback would make - // semantic result #0 and keyword result #0 fuse as if they were the same message + // source-qualified: the index is per-retriever, so a bare index would fuse unrelated candidates _id: msgId || `intelligent-${source}-${index}`, rid, msgId, diff --git a/packages/ai-search/src/types.ts b/packages/ai-search/src/types.ts index 8882de9626a46..20ca0cca4bbdb 100644 --- a/packages/ai-search/src/types.ts +++ b/packages/ai-search/src/types.ts @@ -59,16 +59,11 @@ export type IntelligentSearchCandidate = { rid?: string; msgId?: string; pipelineText: string; - /** - * Normalized cosine similarity, set only for semantic candidates. Keyword candidates deliberately - * carry no score: the pipeline reports their full-text rank in the same field, and it is not a - * similarity, so there is no honest value to show. - */ + /** Normalized cosine similarity. Unset for keyword candidates, which have no comparable score. */ score?: number; semanticSimilarity?: number; semanticDistance?: number; source?: IntelligentSearchCandidateSource; - /** message timestamp reported by the pipeline, used by the temporal rerank stage */ ts?: string; }; @@ -79,7 +74,7 @@ export type FusedIntelligentSearchCandidate = IntelligentSearchCandidate & { }; export type TemporalRerankOptions = { - /** 0 disables the boost, 100 gives the freshest candidate double the relevance score */ + /** 0 disables the boost, 100 doubles the freshest candidate's score */ recencyWeight: number; halfLifeDays: number; now?: Date; From fb90fed41fe75e2dae19671dd50cc2792c97e92c Mon Sep 17 00:00:00 2001 From: Dnouv Date: Fri, 11 Sep 2026 23:29:51 +0800 Subject: [PATCH 06/10] chore: shorten AI Search changeset to match repo conventions --- .changeset/hybrid-ai-search-retrieval.md | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/.changeset/hybrid-ai-search-retrieval.md b/.changeset/hybrid-ai-search-retrieval.md index cb9dfc8f1d03d..2c914e0cc7bc2 100644 --- a/.changeset/hybrid-ai-search-retrieval.md +++ b/.changeset/hybrid-ai-search-retrieval.md @@ -6,15 +6,4 @@ '@rocket.chat/meteor': minor --- -Adds hybrid retrieval and optional temporal reranking to AI Search. - -A single **Search balance** setting (0-100) now controls retrieval: 0 searches by keyword only, 100 by -meaning only, and anything in between runs both retrievers in parallel and fuses them with weighted -Reciprocal Rank Fusion. Fusion works on rank positions, so the retrievers' incompatible score scales are -never compared directly. - -The minimum semantic similarity guardrail now applies only to semantic candidates, so an exact match on -an error code or ticket id is no longer discarded for being semantically unremarkable. - -An optional **Recency boost** reranks results by age after relevance ranking, using an exponential decay -with a 30-day half-life. It is disabled by default and leaves ranking unchanged until an admin opts in. +Adds hybrid retrieval to AI Search. A single search balance setting decides how much semantic retrieval contributes relative to keyword search, so a workspace can find messages by meaning without losing exact matches on error codes, ticket ids or function names. An optional recency boost, disabled by default, promotes newer messages after relevance ranking. From cea79322d24c145362ce7af3d93a748b3b2e891c Mon Sep 17 00:00:00 2001 From: Dnouv Date: Fri, 11 Sep 2026 23:53:09 +0800 Subject: [PATCH 07/10] docs: trim AI Search docs to durable reference Moves the benchmark numbers, tuning rationale and pipeline limitations to the PR description. They are point-in-time findings about one pipeline deployment, not something the codebase should carry and have to keep current. Also picks up review fixes: the candidate cap now sits above the largest page size, so a 50-result request still over-fetches rather than returning a short page once permission filtering runs, and the pipeline request's `mode` narrows to a single retriever, since passing `hybrid` there silently produced a semantic request. --- .../services/ai-search/service.tests.ts | 3 +- docs/features/ai-search-hybrid-benchmark.md | 129 ------------------ docs/features/ai-search-hybrid.md | 120 +++++----------- packages/ai-search/src/constants.ts | 5 +- packages/ai-search/src/types.ts | 3 +- 5 files changed, 41 insertions(+), 219 deletions(-) delete mode 100644 docs/features/ai-search-hybrid-benchmark.md diff --git a/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts b/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts index 1dbab4ffddf87..f8c238f1bbed5 100644 --- a/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts +++ b/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts @@ -341,8 +341,9 @@ describe('AISearchService', () => { await service.search({ query: 'fruit', userId: 'user-id', limit: 9 }); expect(JSON.parse(serverFetch.lastCall.args[1].body).params.k).to.equal(27); + // the cap stays above the largest page so permission filtering cannot shorten it await service.search({ query: 'fruit', userId: 'user-id', limit: 50 }); - expect(JSON.parse(serverFetch.lastCall.args[1].body).params.k).to.equal(50); + expect(JSON.parse(serverFetch.lastCall.args[1].body).params.k).to.equal(100); }); it('keeps a full page of hybrid results when fusion candidates are not visible', async () => { diff --git a/docs/features/ai-search-hybrid-benchmark.md b/docs/features/ai-search-hybrid-benchmark.md deleted file mode 100644 index 713f2af15e3e4..0000000000000 --- a/docs/features/ai-search-hybrid-benchmark.md +++ /dev/null @@ -1,129 +0,0 @@ -# AI Search hybrid retrieval: offline benchmark - -Measurements behind the hybrid search defaults. Re-run these before changing -`INTELLIGENT_SEARCH_CANDIDATE_MULTIPLIER`, `MIN_INTELLIGENT_SEARCH_CANDIDATES`, -`INTELLIGENT_SEARCH_RRF_CONSTANT`, or the shipped setting defaults. - -## Latency - -Hybrid issues both retriever requests with `Promise.all`, so its latency is the *slower* branch, not the -sum. At the shipped pool sizes both branches sit in the same 550-700 ms p50 band, so hybrid costs -roughly one retrieval, plus fusion and reranking which are in-memory over at most 100 candidates. - -The extremes of the balance (`0` and `100`) issue **one** request, not two — the unused retriever is -never called. - -## Method - -- **Corpus**: 547 synthetic Rocket.Chat messages ingested into a QA Intelligent Search pipeline — - 22 judged messages plus 525 topically adjacent distractors, so that top-k retrieval is actually - selective. Messages carry `room_id`, `username` and `timestamp` metadata exactly as production does. -- **Queries**: 20 judged queries in four families: - - `lexical` — exact identifiers (`CVE-2025-1337`, `SUP-4471`, `normalizeMessagesForUser`) - - `conceptual` — paraphrases with minimal lexical overlap with their targets - - `mixed` — an identifier plus a concept - - `recency` — two near-duplicate messages where only the newer one is correct -- **Grades**: 0-3 per (query, message). Metrics are nDCG@10, MRR@10 (first hit graded ≥ 2), recall@10. -- **Harness**: replicates `packages/ai-search/src/fusion.ts` exactly, including the `w = 0` / `w = 100` - short-circuits, and reuses one retrieval per branch across the whole sweep. - -Absolute numbers are only meaningful relative to each other: the corpus is synthetic and small. - -## Semantic weight sweep (candidate pool 50) - -| w | nDCG@10 | MRR@10 | R@10 | conceptual | lexical | mixed | recency | -| --- | --- | --- | --- | --- | --- | --- | --- | -| 0 (keyword only) | 0.3455 | 0.3000 | 0.450 | 0.0000 | 0.7891 | 0.2774 | 0.0000 | -| 10-40 | 0.6814 | 0.5892 | 0.933 | 0.6303 | 0.7891 | 0.6412 | 0.5582 | -| 50 | 0.7112 | 0.6392 | 0.933 | 0.6303 | 0.8418 | 0.6865 | 0.5582 | -| **60** | **0.7152** | 0.6392 | 0.933 | 0.6303 | 0.8418 | 0.7026 | 0.5582 | -| 70-90 | 0.7091 | 0.6392 | 0.933 | 0.6303 | 0.8418 | 0.6780 | 0.5582 | -| 100 (semantic only) | 0.7091 | 0.6392 | 0.933 | 0.6303 | 0.8418 | 0.6780 | 0.5582 | - -- Keyword-only is not a viable default: it scores **0.0** on conceptual and recency queries. -- Hybrid beats semantic-only, and the entire gain sits in `mixed` queries (0.7026 vs 0.6780, +3.6% - relative) — exactly the family hybrid exists for. Other families are unchanged. -- The curve is a step function rather than a smooth slope, because the keyword branch returns very few - rows (see the limitation below). The plateau from 50-90 means the setting is forgiving. - -**Shipped default: 50.** 60 measured marginally higher (+0.6% relative), which is well inside the noise -of a 20-query synthetic set. 50 is the neutral, defensible midpoint; revisit with judged production -queries rather than promoting 60 on this evidence. - -## Candidate pool: the quality/latency frontier - -Quality alone would pick a pool of 50. Latency says otherwise. Pipeline round-trip measured over -20 queries × 3 repetitions: - -| pool (k) | semantic nDCG@10 | best hybrid nDCG@10 | semantic p50 | semantic p95 | -| --- | --- | --- | --- | --- | -| 5 (pre-feature default) | 0.6377 | 0.6438 | 588 ms | 820 ms | -| **20** | **0.6819** | **0.6881** | **584 ms** | 709 ms | -| 50 | 0.7091 | 0.7152 | 910 ms | 1342 ms | -| 100 | 0.6925 | 0.6987 | 1490 ms | 2638 ms | - -- **20 is free**: it costs the same as the old pool of 5 (584 ms vs 588 ms p50) and lifts nDCG@10 by - **6.9%**. Below ~20 the pipeline's vector index is clearly not searching hard enough. -- **50 is not free**: +4.0% nDCG for **+56% latency**. Wrong trade for navbar typeahead, which fires on - every debounced keystroke. -- **100 is strictly worse**: slower *and* lower quality than 50. - -Hence `MIN_INTELLIGENT_SEARCH_CANDIDATES = 20`, `MAX_INTELLIGENT_SEARCH_CANDIDATES = 50`, multiplier ×3. -The navbar (`limit` 5) lands on 20 — same latency as before the feature, better relevance. The search -page (`limit` 9, growing to 50 on *Show more*) scales to the 50 cap, where the extra latency is paid by a -deliberate full-page search rather than by typeahead. - -Keyword-branch latency is flat across k (547-666 ms p50), so the pool size is a semantic-side cost. - -## Temporal boost sweep (w = 60, candidate pool 50) - -nDCG@10, by recency weight and half-life: - -| recency weight | half-life 7d | half-life 30d | half-life 90d | -| --- | --- | --- | --- | -| 0 (off) | 0.7152 | 0.7152 | 0.7152 | -| 10 | 0.7422 | 0.7471 | 0.7296 | -| 25 | 0.7272 | **0.7507** | 0.7510 | -| 50 | 0.7012 | 0.7398 | 0.7495 | -| 75 | 0.6677 | 0.7339 | **0.7524** | -| 100 | 0.6581 | 0.7215 | 0.7505 | - -At weight 25 / half-life 30: overall nDCG@10 **0.7152 → 0.7507 (+5.0%)**, recency queries -**0.5582 → 0.7685 (+37.7%)**, and lexical queries are **completely unaffected** (0.8418 throughout) — -the boost never displaces an exact-identifier match. - -Aggressive settings are actively harmful: weight 100 with a 7-day half-life drops conceptual queries -from 0.6303 to 0.4695. Short half-lives are sharp and unforgiving; 30-90 days are stable. - -**Shipped default: weight 0 (disabled).** Hybrid relevance ships first and ranking stays unchanged -unless an admin opts in. Recommended starting point when enabling: **weight 25**. - -The half-life is **not** an admin setting - it is fixed at 30 days. Across the useful weight range the -30-day and 90-day columns differ by well under 1% nDCG (0.7507 vs 0.7510 at weight 25), so the knob buys -no reachable quality. Only the 7-day column behaves differently, and it behaves *worse*. If half-life -ever needs to move, change the constant on the evidence of a fresh sweep rather than delegating it to -admins. - -## Backend limitations found while benchmarking - -Both are pipeline-side and worth raising with the Intelligent Search team; neither is fixable in -Rocket.Chat. - -1. **Full-text search is strict AND.** `kubectl ramen` and `kubectl zzzznotaword` both return zero rows. - Any conversational multi-word query therefore returns nothing from the keyword branch, which is why - the keyword branch contributes to so few queries above. Keyword retrieval is a precision aid for - identifier-style queries, not a recall workhorse. -2. **Full-text recall is incomplete.** Probing every content token of every document against the index, - only **83%** (191/230) retrieved their own document. Misses include ordinary content words — - `webhook`, `stale`, `rate`, `connection`, `cluster`, `nodes`, `login`, `mobile`, `Safari` — and are - reproducible across re-ingestion of the same text, so they are not a one-off indexing glitch. - -If full-text recall improves, re-run the weight sweep: the keyword branch would carry far more weight -and the optimum would likely move. - -## Reproducing - -The harness is not checked in — it depends on live pipeline credentials. It ingests a generated corpus -via `POST /pipelines/{id}/documents`, queries `POST /pipelines/{id}/search` once per branch per query, -then replays `fusion.ts` locally across the parameter grid. Point it at a disposable pipeline; it writes -several hundred documents. diff --git a/docs/features/ai-search-hybrid.md b/docs/features/ai-search-hybrid.md index 3ae7e40914a1a..9fc1a6d6604b3 100644 --- a/docs/features/ai-search-hybrid.md +++ b/docs/features/ai-search-hybrid.md @@ -1,9 +1,9 @@ # AI Search: hybrid retrieval and temporal reranking -## Overview +## Retrieval -AI Search retrieves messages from an external Intelligent Search pipeline. Both retrievers are the *same* -pipeline endpoint (`POST /pipelines/{id}/search`), distinguished only by the request body: +Both retrievers are the *same* pipeline endpoint (`POST /pipelines/{id}/search`), distinguished only by +the request body: | Retriever | Pipeline request | Threshold sent | | --- | --- | --- | @@ -18,69 +18,26 @@ Which of them runs is decided by a single setting, `AI_Intelligent_Search_Semant | `1`-`99` | both in parallel, fused with weighted RRF | | `100` | semantic only - the keyword retriever is never called | -There is deliberately **no separate "search mode" setting**: the balance already expresses every mode, -and a second control would only let the two disagree. +There is deliberately no separate "search mode" setting: the balance already expresses every mode, and a +second control would only let the two disagree. A caller can pin an endpoint of the range per request +with the `searchType` query parameter on `GET /v1/ai.search` (`keyword` maps to 0, `semantic` to 100, +`hybrid` to whatever the setting says). -A caller can pin an endpoint of that range per request with the `searchType` query parameter on -`GET /v1/ai.search` (`keyword` maps to 0, `semantic` to 100, `hybrid` to whatever the setting says), -which is useful for evaluation without changing workspace configuration. - -## Pipeline retrieval - -``` - QUERY - │ - Apply hard filters - room scope / username / date range - │ - ┌───────────┴───────────┐ - ▼ ▼ - Full-text search Semantic search - k = candidate pool k = candidate pool - │ │ - │ similarity guardrail - │ │ - └──────────┬────────────┘ - ▼ - Weighted RRF - │ - relevance - ▼ - Temporal boost - ▼ - visibility filtering - ▼ - Top N -``` - -Room scoping, username and date filters are applied by the pipeline itself -(`buildIntelligentSearchPipelineFilters`), so they constrain both branches identically. - -## Why fusion happens in Rocket.Chat, not in the pipeline - -The pipeline advertises a native `type: "hybrid"` placeholder, but it returns -`501 Hybrid placeholder is not implemented yet`, and its parameter schema exposes only `k` — there is no -weight. Client-side fusion is therefore both necessary today and the only way to offer an admin-tunable -balance. +Fusion happens here rather than in the pipeline because the pipeline's own `type: "hybrid"` placeholder +returns `501 Hybrid placeholder is not implemented yet`, and its parameter schema exposes no weight. ## Score conventions -The two retrievers report scores on incompatible scales, verified empirically against a live pipeline: - -- Semantic `score` is a **cosine distance**: *lower is better*. The best hit for a well-matched query - scored `0.44`, an unrelated message `0.81`. -- Full-text `score` is a **rank**: *higher is better*. +The two retrievers report scores on incompatible scales: -`normalizeIntelligentSearchCandidates` converts distance to similarity (`similarity = 1 - distance`) for -display and for the similarity guardrail, and does so **only for semantic candidates**. Both retrievers -report their number in the same `score` field, so reading a keyword hit's rank as a distance would invert -it and fabricate a confident similarity — the best lexical hit would display the lowest score. Keyword -candidates therefore carry no `score` at all, and the results UI renders a match percentage only where one -genuinely exists. In a fused list that means the badge appears on semantically-found hits and is absent on -keyword-only hits — an honest gap rather than a fabricated number. +- semantic `score` is a **cosine distance** - *lower* is better +- keyword `score` is a **full-text rank** - *higher* is better -Fusion deliberately never compares the two raw scores — it works on **rank positions only**, which is -what makes the incompatible scales a non-problem. +`normalizeIntelligentSearchCandidates` converts distance to similarity (`1 - distance`) for display and +for the guardrail, and does so only for semantic candidates. Both retrievers report their number in the +same `score` field, so reading a keyword hit's rank as a distance would invert it and fabricate a +confident similarity. Keyword candidates therefore carry no `score`, and the results UI renders a match +percentage only where one genuinely exists. ## Weighted RRF @@ -90,20 +47,19 @@ For a document `d`, with `w = AI_Intelligent_Search_Semantic_Weight / 100` and ` score(d) = (1 - w) / (C + rank_fulltext(d)) + w / (C + rank_semantic(d)) ``` -A branch that did not return `d` contributes nothing. `w = 0` and `w = 100` short-circuit to a single -retriever, so the other branch is never even requested. +Fusion works on **rank positions only**, which is what makes the incompatible scales a non-problem. A +branch that did not return `d` contributes nothing. The two branches are issued with `Promise.allSettled`, not `Promise.all`: if one retriever throws (network failure, or the pipeline's 10s timeout) the search degrades to the surviving retriever rather than returning nothing. Only a double failure propagates. -`C` and the candidate pool size are implementation parameters and are intentionally **not** admin -settings. +`C` and the candidate pool size are implementation parameters and are intentionally not admin settings. ## The similarity guardrail `AI_Intelligent_Search_Min_Similarity_Percent` applies **only to semantic candidates**, and only after -retrieval. A keyword hit is never discarded for being semantically unremarkable — that is precisely the +retrieval. A keyword hit is never discarded for being semantically unremarkable - that is precisely the case hybrid search exists to serve (exact error codes, ticket ids, function names). It defaults to `0` (disabled) and should stay that way for most workspaces: a fixed embedding threshold @@ -112,35 +68,27 @@ it as a garbage-result guardrail, not a quality control. ## Temporal reranking -Applied after fusion, so relevance selects the candidates and freshness only reorders them: +Applied after fusion, so relevance selects the candidates and freshness only reorders them. With +`w = AI_Intelligent_Search_Recency_Weight / 100`: ``` -final(d) = score(d) × (1 + recencyWeight × 2^(-ageInDays / halfLifeDays)) +final(d) = score(d) × (1 + w × 2^(-ageInDays / 30)) ``` -`AI_Intelligent_Search_Recency_Weight` (0-100, default **0** = disabled) is the only control. The -half-life is fixed at 30 days (`DEFAULT_INTELLIGENT_SEARCH_RECENCY_HALF_LIFE_DAYS`): offline sweeps found -the 30-90 day range essentially flat, so exposing it would add a setting without adding reachable -quality. +The weight defaults to **0** (disabled). The half-life is fixed at 30 days +(`DEFAULT_INTELLIGENT_SEARCH_RECENCY_HALF_LIFE_DAYS`); offline sweeps found the 30-90 day range +essentially flat, so exposing it would add a setting without adding reachable quality. Timestamps come from the pipeline fragment metadata, so the boost costs no extra database work. Candidates without a usable timestamp keep their relevance score rather than being penalised. -Because the boost is multiplicative and bounded by `1 + recencyWeight`, it can reorder near-ties but -cannot overturn a large relevance gap. +Because the boost is multiplicative and bounded by `1 + w`, it can reorder near-ties but cannot overturn +a large relevance gap. ## Candidate pool -Each branch is asked for more candidates than the caller requested -(`limit × 3`, floored at 50, capped at 100). This exists for two reasons: - -1. fusion needs overlap to work with; -2. results are filtered for visibility and room subscription **after** retrieval, so a pool the size of - the page would return short pages. - -Neither the fusion nor the normalization step truncates before that filtering runs. - -## Benchmark - -See [ai-search-hybrid-benchmark.md](./ai-search-hybrid-benchmark.md) for the offline relevance -measurements behind the defaults. +Each branch is asked for more candidates than the caller requested (`limit × 3`, clamped to +`[20, 100]`). Fusion needs overlap to work with, and results are filtered for visibility and room +subscription **after** retrieval, so a pool the size of the page would return short pages. The cap stays +above `MAX_INTELLIGENT_SEARCH_RESULTS` for that reason. Neither fusion nor normalization truncates +before that filtering runs. diff --git a/packages/ai-search/src/constants.ts b/packages/ai-search/src/constants.ts index 66fde9e28c4bb..c5a4000af50ba 100644 --- a/packages/ai-search/src/constants.ts +++ b/packages/ai-search/src/constants.ts @@ -6,10 +6,11 @@ export const AI_SEARCH_FILTER_SUGGESTION_LIMIT = 5; export const AI_SEARCH_ROOM_LOOKUP_LIMIT = 20; export const MAX_INTELLIGENT_SEARCH_RESULTS = 50; // Per-retriever candidate pool. Internal, and sized on a measured quality/latency frontier rather than -// quality alone - see docs/features/ai-search-hybrid-benchmark.md before changing these. +// quality alone. The cap must stay above MAX_INTELLIGENT_SEARCH_RESULTS, otherwise the largest page +// over-fetches nothing and permission filtering can return a short page. export const INTELLIGENT_SEARCH_CANDIDATE_MULTIPLIER = 3; export const MIN_INTELLIGENT_SEARCH_CANDIDATES = 20; -export const MAX_INTELLIGENT_SEARCH_CANDIDATES = 50; +export const MAX_INTELLIGENT_SEARCH_CANDIDATES = 100; export const INTELLIGENT_SEARCH_RRF_CONSTANT = 60; export const DEFAULT_INTELLIGENT_SEARCH_SEMANTIC_WEIGHT = 50; export const DEFAULT_INTELLIGENT_SEARCH_RECENCY_HALF_LIFE_DAYS = 30; diff --git a/packages/ai-search/src/types.ts b/packages/ai-search/src/types.ts index 20ca0cca4bbdb..97694bd662d61 100644 --- a/packages/ai-search/src/types.ts +++ b/packages/ai-search/src/types.ts @@ -100,5 +100,6 @@ export type IntelligentSearchPipelineRequest = { limit: number; fetch: AIServiceFetch; logger?: AIServiceLogger; - mode?: IntelligentSearchType; + /** a single request targets one retriever; `hybrid` is resolved before reaching this layer */ + mode?: IntelligentSearchCandidateSource; }; From 08733e10a1d0a253fb2b46c771f9500817ab1a43 Mon Sep 17 00:00:00 2001 From: Dnouv Date: Sat, 12 Sep 2026 20:35:11 +0800 Subject: [PATCH 08/10] fix: preserve hybrid search candidates and handle retrieval failures --- .../server/services/ai-search/service.ts | 11 ++-- .../services/ai-search/service.tests.ts | 50 +++++++++++++++++++ docs/features/ai-search-hybrid.md | 24 +++++---- packages/ai-search/src/fusion.spec.ts | 20 +++++++- packages/ai-search/src/fusion.ts | 21 ++++---- .../ai-search/src/intelligentSearch.spec.ts | 25 ++++++++-- packages/ai-search/src/intelligentSearch.ts | 10 +++- 7 files changed, 131 insertions(+), 30 deletions(-) diff --git a/apps/meteor/server/services/ai-search/service.ts b/apps/meteor/server/services/ai-search/service.ts index 7b652fe83c4d9..0959173c07a12 100644 --- a/apps/meteor/server/services/ai-search/service.ts +++ b/apps/meteor/server/services/ai-search/service.ts @@ -272,11 +272,16 @@ export class AISearchService extends ServiceClass implements IAISearchService { return toRankedCandidates(semanticCandidates); } - return fuseCandidatesWithWeightedRRF(semanticCandidates, keywordCandidates, semanticWeight, candidateLimit); + // Preserve the union until visibility filtering and temporal reranking have run. + return fuseCandidatesWithWeightedRRF( + semanticCandidates, + keywordCandidates, + semanticWeight, + semanticCandidates.length + keywordCandidates.length, + ); } - // Larger than the requested page so fusion has overlap to work with and permission filtering below - // cannot eat into the page. + // Over-fetch to improve fusion overlap and reduce short pages after permission filtering. private getSearchCandidateLimit(requestedLimit: number): number { const scaledLimit = Math.max(requestedLimit, AI_SEARCH_PAGE_SIZE) * INTELLIGENT_SEARCH_CANDIDATE_MULTIPLIER; diff --git a/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts b/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts index f8c238f1bbed5..44248277266ed 100644 --- a/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts +++ b/apps/meteor/tests/unit/server/services/ai-search/service.tests.ts @@ -380,6 +380,25 @@ describe('AISearchService', () => { expect(results.map(({ _id }: { _id: string }) => _id)).to.deep.equal(['visible-b', 'visible-a']); }); + it('retains the other branch beyond the candidate cap when higher-ranked messages are inaccessible', async () => { + cachedSettings.get.callsFake((key: string) => (key === 'AI_Intelligent_Search_Semantic_Weight' ? 90 : settings[key])); + serverFetch.callsFake(async (_url: string, options: { body: string }) => { + const { type, params } = JSON.parse(options.body); + const results = Array.from({ length: params.k }, (_, index) => ({ + metadata: { room_id: type === 'similarity' ? 'forbidden' : 'allowed', msg_id: `${type}-${index}` }, + score: 0.2, + })); + return { ok: true, status: 200, json: async () => ({ results }), text: async () => '' }; + }); + Messages.findVisibleByIds.callsFake((msgIds: string[]) => + cursor(msgIds.map((_id) => ({ _id, rid: _id.startsWith('similarity-') ? 'forbidden' : 'allowed', msg: _id }))), + ); + + const results = await createService().search({ query: 'fruit', userId: 'user-id', limit: 2 }); + + expect(results.map(({ _id }: { _id: string }) => _id)).to.deep.equal(['search-0', 'search-1']); + }); + it('promotes fresher messages once the recency boost is enabled', async () => { const timestamps: Record = { stale: '2020-01-01T12:00:00.000Z', @@ -444,6 +463,37 @@ describe('AISearchService', () => { ); }); + for (const failedType of ['similarity', 'search']) { + it(`serves the surviving retriever when ${failedType} returns HTTP 503`, async () => { + cachedSettings.get.callsFake((key: string) => (key === 'AI_Intelligent_Search_Semantic_Weight' ? 50 : settings[key])); + serverFetch.callsFake(async (_url: string, options: { body: string }) => { + const failed = JSON.parse(options.body).type === failedType; + return { + ok: !failed, + status: failed ? 503 : 200, + json: async () => ({ results: [{ id: 'allowed-msg', score: 0.2 }] }), + text: async () => '', + }; + }); + + const results = await createService().search({ query: 'fruit', userId: 'user-id' }); + + expect(results.map(({ _id }: { _id: string }) => _id)).to.deep.equal(['allowed-msg']); + }); + } + + it('rejects when both retrievers return HTTP errors', async () => { + cachedSettings.get.callsFake((key: string) => (key === 'AI_Intelligent_Search_Semantic_Weight' ? 50 : settings[key])); + serverFetch.resolves({ ok: false, status: 503, text: async () => '' }); + + await createService() + .search({ query: 'fruit', userId: 'user-id' }) + .then( + () => expect.fail('expected the search to reject'), + (error: Error) => expect(error.message).to.equal('Intelligent search pipeline returned HTTP 503'), + ); + }); + it('lets an explicit searchType pin an endpoint of the balance without an admin change', async () => { cachedSettings.get.callsFake((key: string) => (key === 'AI_Intelligent_Search_Semantic_Weight' ? 50 : settings[key])); serverFetch.resolves({ diff --git a/docs/features/ai-search-hybrid.md b/docs/features/ai-search-hybrid.md index 9fc1a6d6604b3..9e1a1cf9a6591 100644 --- a/docs/features/ai-search-hybrid.md +++ b/docs/features/ai-search-hybrid.md @@ -51,20 +51,23 @@ Fusion works on **rank positions only**, which is what makes the incompatible sc branch that did not return `d` contributes nothing. The two branches are issued with `Promise.allSettled`, not `Promise.all`: if one retriever throws -(network failure, or the pipeline's 10s timeout) the search degrades to the surviving retriever rather -than returning nothing. Only a double failure propagates. +(HTTP error, network failure, or the pipeline's 10s timeout) the search degrades to the surviving retriever rather +than returning nothing. A double failure rejects the service call. The existing REST handler logs +that error and returns an empty result list. `C` and the candidate pool size are implementation parameters and are intentionally not admin settings. ## The similarity guardrail -`AI_Intelligent_Search_Min_Similarity_Percent` applies **only to semantic candidates**, and only after -retrieval. A keyword hit is never discarded for being semantically unremarkable - that is precisely the +`AI_Intelligent_Search_Min_Similarity_Percent` applies **only to semantic candidates**, through the +pipeline request's distance threshold and again after retrieval. A keyword hit is never discarded for +being semantically unremarkable - that is precisely the case hybrid search exists to serve (exact error codes, ticket ids, function names). It defaults to `0` (disabled) and should stay that way for most workspaces: a fixed embedding threshold is brittle across embedding models, query length, language and corpus, whereas ranking is stable. Treat -it as a garbage-result guardrail, not a quality control. +it as a garbage-result guardrail, not a quality control. If the pipeline omits all similarity metadata, +unscored semantic candidates are preserved for compatibility. ## Temporal reranking @@ -82,13 +85,16 @@ essentially flat, so exposing it would add a setting without adding reachable qu Timestamps come from the pipeline fragment metadata, so the boost costs no extra database work. Candidates without a usable timestamp keep their relevance score rather than being penalised. -Because the boost is multiplicative and bounded by `1 + w`, it can reorder near-ties but cannot overturn -a large relevance gap. +The boost is bounded by `1 + w`, at most doubling a candidate's RRF score. RRF compresses rank +differences, so a high recency weight can move a fresh message substantially up the candidate list. +It cannot overturn an RRF score gap greater than that multiplier. ## Candidate pool Each branch is asked for more candidates than the caller requested (`limit × 3`, clamped to `[20, 100]`). Fusion needs overlap to work with, and results are filtered for visibility and room subscription **after** retrieval, so a pool the size of the page would return short pages. The cap stays -above `MAX_INTELLIGENT_SEARCH_RESULTS` for that reason. Neither fusion nor normalization truncates -before that filtering runs. +above `MAX_INTELLIGENT_SEARCH_RESULTS` for that reason. Each branch retains its highest-ranked fragment +per message. The complete fused union (at most 200 messages) reaches temporal reranking and visibility +filtering before the requested page is selected. Over-fetching reduces short pages but cannot guarantee +a full page if too few retrieved messages remain accessible. diff --git a/packages/ai-search/src/fusion.spec.ts b/packages/ai-search/src/fusion.spec.ts index 8e976add03fe7..0e0389f620162 100644 --- a/packages/ai-search/src/fusion.spec.ts +++ b/packages/ai-search/src/fusion.spec.ts @@ -37,7 +37,7 @@ describe('AI Search fusion helpers', () => { expect(ids(filterSemanticCandidatesByMinimumSimilarity(candidates, 70))).toEqual(['m1', 'm3']); }); - it('keeps candidates that carry no semantic similarity, so keyword hits survive the guardrail', () => { + it('preserves unscored semantic candidates if the pipeline omits similarity metadata', () => { const candidates = [candidate('m1'), candidate('m2', { semanticSimilarity: 0.1 })]; expect(ids(filterSemanticCandidatesByMinimumSimilarity(candidates, 70))).toEqual(['m1']); @@ -64,6 +64,24 @@ describe('AI Search fusion helpers', () => { expect(fused[0].fulltextRank).toBe(1); }); + it('counts a message once per branch and assigns consecutive unique-message ranks', () => { + const fused = fuseCandidatesWithWeightedRRF( + [candidate('repeated'), candidate('repeated'), candidate('shared')], + [candidate('shared')], + 50, + 10, + ); + + expect(ids(fused)).toEqual(['shared', 'repeated']); + expect(fused[0].semanticRank).toBe(2); + expect(fused[1].rrfScore).toBeCloseTo(0.5 / (INTELLIGENT_SEARCH_RRF_CONSTANT + 1), 10); + }); + + it('excludes candidates from a branch with zero weight', () => { + expect(ids(fuseCandidatesWithWeightedRRF(semantic, keyword, 100, 10))).toEqual(['s1', 's2', 'shared']); + expect(ids(fuseCandidatesWithWeightedRRF(semantic, keyword, 0, 10))).toEqual(['shared', 'k1', 'k2']); + }); + it('shifts the ordering as the balance moves towards semantic', () => { expect(ids(fuseCandidatesWithWeightedRRF(semantic, keyword, 10, 3))).toEqual(['shared', 'k1', 'k2']); // 'shared' still leads at 90: its keyword rank 1 tops up an otherwise last-place semantic rank 3 diff --git a/packages/ai-search/src/fusion.ts b/packages/ai-search/src/fusion.ts index e7431a765e564..18c4d93da6f66 100644 --- a/packages/ai-search/src/fusion.ts +++ b/packages/ai-search/src/fusion.ts @@ -46,14 +46,18 @@ export const fuseCandidatesWithWeightedRRF = ( const fused = new Map(); const addBranch = (candidates: IntelligentSearchCandidate[], branch: IntelligentSearchCandidateSource, branchWeight: number): void => { - for (let index = 0; index < candidates.length; index++) { - const candidate = candidates[index]; + if (!branchWeight) { + return; + } + const seen = new Set(); + for (const candidate of candidates) { const candidateId = getCandidateId(candidate); - if (!candidateId) { + if (!candidateId || seen.has(candidateId)) { continue; } + seen.add(candidateId); - const rank = index + 1; + const rank = seen.size; const contribution = branchWeight / (rrfConstant + rank); const existing = fused.get(candidateId); if (!existing) { @@ -67,11 +71,6 @@ export const fuseCandidatesWithWeightedRRF = ( fused.set(candidateId, { ...existing, - ...(branch === 'semantic' && { - score: candidate.score ?? existing.score, - semanticSimilarity: candidate.semanticSimilarity ?? existing.semanticSimilarity, - semanticDistance: candidate.semanticDistance ?? existing.semanticDistance, - }), ts: existing.ts || candidate.ts, rrfScore: existing.rrfScore + contribution, ...(branch === 'semantic' ? { semanticRank: rank } : { fulltextRank: rank }), @@ -118,9 +117,7 @@ export const getRecencyDecay = (ageInDays: number, halfLifeDays: number): number return 2 ** (-Math.max(0, ageInDays) / halfLifeDays); }; -// Multiplicative and bounded by `1 + recencyWeight`, so freshness reorders near-ties but cannot -// overturn a real relevance gap. A missing or unparseable `ts` keeps the relevance score rather than -// being penalised. +// The boost is bounded by `1 + recencyWeight`. Missing or unparseable timestamps keep the RRF score. export const applyTemporalRerank = ( candidates: FusedIntelligentSearchCandidate[], { recencyWeight, halfLifeDays, now = new Date() }: TemporalRerankOptions, diff --git a/packages/ai-search/src/intelligentSearch.spec.ts b/packages/ai-search/src/intelligentSearch.spec.ts index 4a91e2ff48e64..9315033cbb6a8 100644 --- a/packages/ai-search/src/intelligentSearch.spec.ts +++ b/packages/ai-search/src/intelligentSearch.spec.ts @@ -27,6 +27,23 @@ describe('AI Search intelligent search helpers', () => { }); describe('normalizeIntelligentSearchCandidates', () => { + it.each(['semantic', 'keyword'] as const)('counts unique messages toward the %s candidate limit', (source) => { + const results = normalizeIntelligentSearchCandidates( + [ + { id: 'm1', score: 0.1 }, + { id: 'm1', score: 0.2 }, + { id: 'm2', score: 0.3 }, + ], + [], + 2, + undefined, + source, + ); + + expect(results.map(({ msgId }) => msgId)).toEqual(['m1', 'm2']); + expect(results[0].semanticSimilarity).toBe(source === 'semantic' ? 0.9 : undefined); + }); + it('normalizes supported pipeline response shapes and score formats', () => { const results = normalizeIntelligentSearchCandidates( { @@ -99,7 +116,7 @@ describe('AI Search intelligent search helpers', () => { expect(results).toEqual([{ _id: 'm1', rid: 'r1', msgId: 'm1', pipelineText: '', source: 'semantic' }]); }); - it('optionally marks the semantic source for caller-provided source input', () => { + it('marks keyword candidates without adding a semantic score', () => { expect( normalizeIntelligentSearchCandidates( { results: [{ metadata: { room_id: 'r1', msg_id: 'm1' }, text: 'keyword match', score: 0.42 }] }, @@ -314,7 +331,7 @@ describe('AI Search intelligent search helpers', () => { }); }); - it('returns an empty result set for non-2xx pipeline responses', async () => { + it('rejects non-2xx responses so orchestration can distinguish failure from no matches', async () => { const fetch: AIServiceFetch = async () => ({ ok: false, status: 500, @@ -322,7 +339,7 @@ describe('AI Search intelligent search helpers', () => { text: async () => 'failed', }); - const result = await searchIntelligentPipeline({ + const result = searchIntelligentPipeline({ query: 'fruit colors', config: { baseUrl: 'https://pipeline.example.com', @@ -336,7 +353,7 @@ describe('AI Search intelligent search helpers', () => { fetch, }); - expect(result).toEqual([]); + await expect(result).rejects.toThrow('Intelligent search pipeline returned HTTP 500'); }); }); }); diff --git a/packages/ai-search/src/intelligentSearch.ts b/packages/ai-search/src/intelligentSearch.ts index f668840e83dbe..9a9cb79c6cb83 100644 --- a/packages/ai-search/src/intelligentSearch.ts +++ b/packages/ai-search/src/intelligentSearch.ts @@ -152,6 +152,7 @@ export const normalizeIntelligentSearchCandidates = ( const shouldFilterByRoomIds = userRoomIdSet.size > 0; const candidates: IntelligentSearchCandidate[] = []; + const seenMessageIds = new Set(); for (let index = 0; index < rawResults.length && candidates.length < limit; index++) { const result = asRecord(rawResults[index]); const metadata = asRecord(result.metadata); @@ -163,6 +164,13 @@ export const normalizeIntelligentSearchCandidates = ( logger?.debug?.({ msg: 'Intelligent search result filtered: room not in user subscriptions', rid }); continue; } + // A message can have several indexed fragments. Keep its highest-ranked fragment. + if (msgId) { + if (seenMessageIds.has(msgId)) { + continue; + } + seenMessageIds.add(msgId); + } const { semanticDistance, semanticSimilarity } = extractPipelineSimilarityScores(result, metadata, source); const ts = firstString(metadata.timestamp, result.timestamp); @@ -313,7 +321,7 @@ export const searchIntelligentPipeline = async ({ if (!response.ok) { const body = await response.text().catch(() => ''); logger?.warn?.({ msg: 'Intelligent search pipeline returned error', url, status: response.status, bodyLength: body.length }); - return []; + throw new Error(`Intelligent search pipeline returned HTTP ${response.status}`); } const json = await response.json(); From 40c176b2131500754d0fff0bb941c6775a0d1ef9 Mon Sep 17 00:00:00 2001 From: Dnouv Date: Mon, 14 Sep 2026 21:13:31 +0800 Subject: [PATCH 09/10] fix: keep semantic scores at full precision and at their true range Rounding the converted similarity to 4 decimal places could promote a candidate past the minimum-similarity guardrail: a cosine distance of 0.30004 became exactly 0.7 and satisfied a 70% minimum it should have missed. The conversion now keeps full floating-point precision, and the comparison happens on the unrounded value. Widens the accepted ranges to what cosine actually spans - similarity [-1, 1], distance [0, 2] - and drops the heuristic that divided any value above 1 by 100 on the assumption it was a percentage. That heuristic would have read a genuine distance of 1.5, a poor match, as 0.015 and reported it as 98.5% similar. The UI-facing score is clamped to [0, 1] separately, so a negative similarity displays as no match rather than as a nonsensical percentage. The live pipeline has not been observed returning a distance above 0.9771, so this is hardening rather than a fix for current behaviour; the precision change above is the part that alters a real outcome. Adds coverage for the score mathematics, the threshold boundary, and the fused score across every supported integer weight. --- docs/features/ai-search-hybrid.md | 57 ++++++++++++++----- packages/ai-search/src/fusion.spec.ts | 32 +++++++++++ packages/ai-search/src/fusion.ts | 2 +- .../ai-search/src/intelligentSearch.spec.ts | 35 +++++++++++- packages/ai-search/src/intelligentSearch.ts | 19 ++----- packages/ai-search/src/types.ts | 4 +- 6 files changed, 120 insertions(+), 29 deletions(-) diff --git a/docs/features/ai-search-hybrid.md b/docs/features/ai-search-hybrid.md index 9e1a1cf9a6591..82a7cac253e2e 100644 --- a/docs/features/ai-search-hybrid.md +++ b/docs/features/ai-search-hybrid.md @@ -20,12 +20,9 @@ Which of them runs is decided by a single setting, `AI_Intelligent_Search_Semant There is deliberately no separate "search mode" setting: the balance already expresses every mode, and a second control would only let the two disagree. A caller can pin an endpoint of the range per request -with the `searchType` query parameter on `GET /v1/ai.search` (`keyword` maps to 0, `semantic` to 100, +with the `searchType` query parameter on `GET /api/v1/ai.search` (`keyword` maps to 0, `semantic` to 100, `hybrid` to whatever the setting says). -Fusion happens here rather than in the pipeline because the pipeline's own `type: "hybrid"` placeholder -returns `501 Hybrid placeholder is not implemented yet`, and its parameter schema exposes no weight. - ## Score conventions The two retrievers report scores on incompatible scales: @@ -39,6 +36,11 @@ same `score` field, so reading a keyword hit's rank as a distance would invert i confident similarity. Keyword candidates therefore carry no `score`, and the results UI renders a match percentage only where one genuinely exists. +Cosine similarity spans `[-1, 1]` and cosine distance spans `[0, 2]`. A distance of `1.2` means +similarity `-0.2`. Scores use unit-scale values and are clamped to their mathematical ranges. +Conversion retains full floating-point precision for filtering. The UI-facing `score` is separately +clamped to `[0, 1]`. + ## Weighted RRF For a document `d`, with `w = AI_Intelligent_Search_Semantic_Weight / 100` and `C = 60`: @@ -50,6 +52,19 @@ score(d) = (1 - w) / (C + rank_fulltext(d)) + w / (C + rank_semantic(d)) Fusion works on **rank positions only**, which is what makes the incompatible scales a non-problem. A branch that did not return `d` contributes nothing. +Ranks start at 1 in each deduplicated list, after semantic threshold filtering. Each message contributes +at most once per branch. Equal fused scores are ordered by semantic rank, then keyword rank. This is a +deterministic tie-break, with a semantic preference on exact ties. + +For nonempty results the weighted score lies in `(0, 1/61]`. At weight 50, it is half the usual +unweighted two-list RRF sum; this constant scaling leaves both ordering and multiplicative recency +reranking unchanged. For example, semantic rank 3 plus keyword rank 1 gives +`0.5/63 + 0.5/61 = 0.0161332`, ahead of a semantic-only rank-1 hit at `0.5/61 = 0.0081967`. + +Single-retriever modes and failure fallback use `1/(60 + rank)`. Rescaling the surviving branch to +weight 1 preserves its ordering, including after the multiplicative recency boost. RRF scores are +ranking values, not match probabilities, and the balance is not a quota for result counts. + The two branches are issued with `Promise.allSettled`, not `Promise.all`: if one retriever throws (HTTP error, network failure, or the pipeline's 10s timeout) the search degrades to the surviving retriever rather than returning nothing. A double failure rejects the service call. The existing REST handler logs @@ -61,33 +76,41 @@ that error and returns an empty result list. `AI_Intelligent_Search_Min_Similarity_Percent` applies **only to semantic candidates**, through the pipeline request's distance threshold and again after retrieval. A keyword hit is never discarded for -being semantically unremarkable - that is precisely the -case hybrid search exists to serve (exact error codes, ticket ids, function names). +being semantically unremarkable (for example, exact error codes, ticket ids, and function names). It defaults to `0` (disabled) and should stay that way for most workspaces: a fixed embedding threshold is brittle across embedding models, query length, language and corpus, whereas ranking is stable. Treat it as a garbage-result guardrail, not a quality control. If the pipeline omits all similarity metadata, unscored semantic candidates are preserved for compatibility. +For an enabled minimum `p`, eligibility is `similarity >= p/100`, equivalently `distance <= 1-p/100`. +No candidate-score rounding occurs before that comparison. A minimum of zero skips the local guardrail; +the semantic pipeline request retains its existing distance threshold of `1`. + ## Temporal reranking -Applied after fusion, so relevance selects the candidates and freshness only reorders them. With -`w = AI_Intelligent_Search_Recency_Weight / 100`: +Applied to the complete fused pool before the final page is selected. With +`w = AI_Intelligent_Search_Recency_Weight / 100` and age measured in elapsed 24-hour days: ``` -final(d) = score(d) × (1 + w × 2^(-ageInDays / 30)) +age(d) = max(0, (now - timestamp(d)) / millisecondsPerDay) +final(d) = score(d) × (1 + w × 2^(-age(d) / 30)) ``` The weight defaults to **0** (disabled). The half-life is fixed at 30 days -(`DEFAULT_INTELLIGENT_SEARCH_RECENCY_HALF_LIFE_DAYS`); offline sweeps found the 30-90 day range -essentially flat, so exposing it would add a setting without adding reachable quality. +(`DEFAULT_INTELLIGENT_SEARCH_RECENCY_HALF_LIFE_DAYS`). The decay is 1 at age zero, 0.5 at 30 days, +and 0.25 at 60 days. It is the additional boost, not the entire ranking score, that halves. Timestamps come from the pipeline fragment metadata, so the boost costs no extra database work. -Candidates without a usable timestamp keep their relevance score rather than being penalised. +Candidates without a usable timestamp keep their relevance score. Future timestamps have age zero, +and ties preserve the incoming fused order. The adjusted score is used for ordering only; `rrfScore` +continues to represent the pre-boost fusion score. The boost is bounded by `1 + w`, at most doubling a candidate's RRF score. RRF compresses rank differences, so a high recency weight can move a fresh message substantially up the candidate list. -It cannot overturn an RRF score gap greater than that multiplier. +For scores `sA > sB > 0`, B cannot overtake A if `sA/sB > 1+w`. This is a ratio bound, not an +absolute score difference or rank bound. For example, at weight 100 a fresh rank-60 result has score +`2/120`, which can exceed a sufficiently old rank-1 result at approximately `1/61`. ## Candidate pool @@ -98,3 +121,11 @@ above `MAX_INTELLIGENT_SEARCH_RESULTS` for that reason. Each branch retains its per message. The complete fused union (at most 200 messages) reaches temporal reranking and visibility filtering before the requested page is selected. Over-fetching reduces short pages but cannot guarantee a full page if too few retrieved messages remain accessible. + +This is fusion over truncated retrieval lists, not the entire corpus. Changing the requested page size +can change the candidate pool, overlap, and final order; it is not a stable pagination snapshot. + +## References + +- [Original RRF paper](https://cormack.uwaterloo.ca/cormacksigir09-rrf.pdf) +- [Cosine distance definition](https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.distance.cosine.html) diff --git a/packages/ai-search/src/fusion.spec.ts b/packages/ai-search/src/fusion.spec.ts index 0e0389f620162..e289ee5ba7c2f 100644 --- a/packages/ai-search/src/fusion.spec.ts +++ b/packages/ai-search/src/fusion.spec.ts @@ -95,6 +95,29 @@ describe('AI Search fusion helpers', () => { expect(top.rrfScore).toBeCloseTo(0.6 / (k + 1), 10); }); + it('matches the weighted sum and score bounds across every supported integer weight', () => { + for (let weight = 0; weight <= 100; weight++) { + const fused = fuseCandidatesWithWeightedRRF(semantic, keyword, weight, 10); + for (const result of fused) { + const semanticRank = semantic.findIndex(({ msgId }) => msgId === result.msgId) + 1; + const keywordRank = keyword.findIndex(({ msgId }) => msgId === result.msgId) + 1; + const expected = + (semanticRank ? weight / 100 / (60 + semanticRank) : 0) + (keywordRank ? (1 - weight / 100) / (60 + keywordRank) : 0); + + expect(result.rrfScore).toBeCloseTo(expected, 14); + expect(result.rrfScore).toBeGreaterThan(0); + expect(result.rrfScore).toBeLessThanOrEqual(1 / 61); + } + } + }); + + it('breaks equal-score ties by semantic rank deterministically', () => { + const fused = fuseCandidatesWithWeightedRRF([candidate('a'), candidate('b')], [candidate('b'), candidate('a')], 50, 2); + + expect(fused[0].rrfScore).toBe(fused[1].rrfScore); + expect(ids(fused)).toEqual(['a', 'b']); + }); + it('degrades to the populated branch when the other retriever returns nothing', () => { expect(ids(fuseCandidatesWithWeightedRRF(semantic, [], 50, 10))).toEqual(['s1', 's2', 'shared']); expect(ids(fuseCandidatesWithWeightedRRF([], keyword, 50, 10))).toEqual(['shared', 'k1', 'k2']); @@ -198,6 +221,15 @@ describe('AI Search fusion helpers', () => { expect(ids(applyTemporalRerank(candidates, { recencyWeight: 100, halfLifeDays: 30, now }))).toEqual(['relevant', 'fresh']); }); + it('can promote rank 60 over rank 1 at maximum recency weight', () => { + const candidates = ranked([ + ['old-first', 1 / 61, daysAgo(3650)], + ['fresh-sixtieth', 1 / 120, daysAgo(0)], + ]); + + expect(ids(applyTemporalRerank(candidates, { recencyWeight: 100, halfLifeDays: 30, now }))).toEqual(['fresh-sixtieth', 'old-first']); + }); + it('leaves candidates without a usable timestamp at their relevance score rather than penalising them', () => { const candidates: FusedIntelligentSearchCandidate[] = [ { ...candidate('no-ts'), rrfScore: 0.02 }, diff --git a/packages/ai-search/src/fusion.ts b/packages/ai-search/src/fusion.ts index 18c4d93da6f66..59bd5e287e2fb 100644 --- a/packages/ai-search/src/fusion.ts +++ b/packages/ai-search/src/fusion.ts @@ -117,7 +117,7 @@ export const getRecencyDecay = (ageInDays: number, halfLifeDays: number): number return 2 ** (-Math.max(0, ageInDays) / halfLifeDays); }; -// The boost is bounded by `1 + recencyWeight`. Missing or unparseable timestamps keep the RRF score. +// The multiplier is bounded by `1 + recencyWeight / 100`. Missing timestamps keep the RRF score. export const applyTemporalRerank = ( candidates: FusedIntelligentSearchCandidate[], { recencyWeight, halfLifeDays, now = new Date() }: TemporalRerankOptions, diff --git a/packages/ai-search/src/intelligentSearch.spec.ts b/packages/ai-search/src/intelligentSearch.spec.ts index 9315033cbb6a8..eb30adf91902d 100644 --- a/packages/ai-search/src/intelligentSearch.spec.ts +++ b/packages/ai-search/src/intelligentSearch.spec.ts @@ -1,3 +1,4 @@ +import { filterSemanticCandidatesByMinimumSimilarity } from './fusion'; import { buildIntelligentSearchPipelineFilters, getSemanticDistanceThreshold, @@ -50,7 +51,7 @@ describe('AI Search intelligent search helpers', () => { results: [ { metadata: { room_id: 'r1', msg_id: 'm1', text: 'metadata text', score: 0.11 } }, { external_identifier: 'r2:m2', content: 'content text', similarity: 0.49 }, - { id: 'm3', rid: 'r3', document: 'document text', distance: 12 }, + { id: 'm3', rid: 'r3', document: 'document text', distance: 0.12 }, { metadata: { room_id: 'r4', msg_id: 'm4', score: null, similarity: '' }, text: 'no numeric score' }, { text: 'missing ids' }, ], @@ -137,6 +138,38 @@ describe('AI Search intelligent search helpers', () => { }); }); + describe('semantic score mathematics', () => { + it.each([0, 0.1743, 0.3098, 0.3807, 1, 1.2, 2])('preserves cosine distance %s without percentage conversion', (distance) => { + const [result] = normalizeIntelligentSearchCandidates([{ id: 'm1', distance }], [], 1); + + expect(result.semanticDistance).toBe(distance); + expect(result.semanticSimilarity).toBeCloseTo(1 - distance, 14); + expect(result.score).toBeCloseTo(Math.max(0, 1 - distance), 14); + }); + + it.each([-1, -0.2, 0, 0.69996, 0.7, 1])('preserves cosine similarity %s and its complementary distance', (similarity) => { + const [result] = normalizeIntelligentSearchCandidates([{ id: 'm1', similarity }], [], 1); + + expect(result.semanticSimilarity).toBe(similarity); + expect(result.semanticDistance).toBeCloseTo(1 - similarity, 14); + expect(result.score).toBe(Math.max(0, similarity)); + }); + + it('does not round a below-threshold candidate into eligibility', () => { + const candidates = normalizeIntelligentSearchCandidates( + [ + { id: 'below', distance: 0.30004 }, + { id: 'boundary', distance: 0.3 }, + { id: 'above', distance: 0.29996 }, + ], + [], + 3, + ); + + expect(filterSemanticCandidatesByMinimumSimilarity(candidates, 70).map(({ msgId }) => msgId)).toEqual(['boundary', 'above']); + }); + }); + describe('keyword candidate scores', () => { it('never reports a full-text rank as a semantic similarity', () => { // 0.2803 is the stronger lexical hit; read as a distance it would display as the weaker one diff --git a/packages/ai-search/src/intelligentSearch.ts b/packages/ai-search/src/intelligentSearch.ts index 9a9cb79c6cb83..dcdf6886b22f9 100644 --- a/packages/ai-search/src/intelligentSearch.ts +++ b/packages/ai-search/src/intelligentSearch.ts @@ -56,14 +56,6 @@ export const normalizeSimilarityPercent = (value: unknown): number => { export const getSemanticDistanceThreshold = (minimumSimilarityPercent: number): number => Number((1 - minimumSimilarityPercent / 100).toFixed(4)); -// Pipeline contract, verified against a live pipeline: `score`/`distance` are cosine distances (lower is -// better), `similarity` values are cosine similarities. Percentages are accepted for provider drift. -const normalizePipelineScore = (value: number): number => { - const normalizedValue = Math.abs(value) > 1 ? value / 100 : value; - - return Math.min(1, Math.max(0, normalizedValue)); -}; - // The keyword retriever reuses `score` for a full-text rank where higher is better, so reading it as a // distance would invert it and fabricate a confident similarity. const extractPipelineSimilarityScores = ( @@ -77,8 +69,8 @@ const extractPipelineSimilarityScores = ( const similarity = firstNumber(result.similarity, metadata.similarity); if (typeof similarity === 'number') { - const semanticSimilarity = normalizePipelineScore(similarity); - const semanticDistance = Number((1 - semanticSimilarity).toFixed(4)); + const semanticSimilarity = Math.min(1, Math.max(-1, similarity)); + const semanticDistance = 1 - semanticSimilarity; return { semanticSimilarity, @@ -88,10 +80,11 @@ const extractPipelineSimilarityScores = ( const distance = firstNumber(result.score, result.distance, metadata.score, metadata.distance); if (typeof distance === 'number') { - const semanticDistance = normalizePipelineScore(distance); + // Cosine distance spans [0, 2]; values above 1 indicate negative similarity. + const semanticDistance = Math.min(2, Math.max(0, distance)); return { - semanticSimilarity: Number((1 - semanticDistance).toFixed(4)), + semanticSimilarity: 1 - semanticDistance, semanticDistance, }; } @@ -182,7 +175,7 @@ export const normalizeIntelligentSearchCandidates = ( pipelineText: firstString(result.text, result.content, result.document, result.page_content, metadata.text) || '', ...(ts && { ts }), ...(typeof semanticSimilarity === 'number' && { - score: semanticSimilarity, + score: Math.max(0, semanticSimilarity), semanticSimilarity, semanticDistance, }), diff --git a/packages/ai-search/src/types.ts b/packages/ai-search/src/types.ts index 97694bd662d61..39f1020413080 100644 --- a/packages/ai-search/src/types.ts +++ b/packages/ai-search/src/types.ts @@ -59,9 +59,11 @@ export type IntelligentSearchCandidate = { rid?: string; msgId?: string; pipelineText: string; - /** Normalized cosine similarity. Unset for keyword candidates, which have no comparable score. */ + /** Display similarity clamped to [0, 1]. Unset for keyword candidates. */ score?: number; + /** Cosine similarity in [-1, 1], retained at full precision for filtering. */ semanticSimilarity?: number; + /** Cosine distance in [0, 2]. */ semanticDistance?: number; source?: IntelligentSearchCandidateSource; ts?: string; From 962a02e3fb4177e9a7b1d840c15ab9b841fb2089 Mon Sep 17 00:00:00 2001 From: Dnouv Date: Mon, 14 Sep 2026 21:29:07 +0800 Subject: [PATCH 10/10] fix: bound the AI Search percentage settings and export AISearchType Both 0-100 settings were registered as plain `int`, so a value outside the range persisted and was only clamped at read time in the service. They are now `range`, which getSettingDefaults gives minValue 0 and maxValue 100 and which checkSettingValueBounds enforces on save, so invalid configuration is rejected rather than silently corrected. It also renders as a slider, which suits a balance control better than a free-text number. AISearchType was the only type in IAISearchService.ts missing from the core-services barrel, so consumers had to deep-import it or restate the union. Also disambiguates an RRF example in the docs that read as a single-retriever score when it was describing a hybrid candidate found by one branch only. --- apps/meteor/server/settings/ai.ts | 4 ++-- docs/features/ai-search-hybrid.md | 3 ++- packages/core-services/src/index.ts | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/meteor/server/settings/ai.ts b/apps/meteor/server/settings/ai.ts index ab1a6819c515a..461c48f5d9b70 100644 --- a/apps/meteor/server/settings/ai.ts +++ b/apps/meteor/server/settings/ai.ts @@ -55,7 +55,7 @@ export const createAISettings = async (): Promise => { await settingsRegistry.add('AI_Intelligent_Search_Semantic_Weight', 50, { group: AI_SETTINGS_GROUP, section: 'Intelligent_Search', - type: 'int', + type: 'range', i18nLabel: 'AI_Intelligent_Search_Semantic_Weight', i18nDescription: 'AI_Intelligent_Search_Semantic_Weight_Description', enterprise: true, @@ -67,7 +67,7 @@ export const createAISettings = async (): Promise => { await settingsRegistry.add('AI_Intelligent_Search_Recency_Weight', 0, { group: AI_SETTINGS_GROUP, section: 'Intelligent_Search', - type: 'int', + type: 'range', i18nLabel: 'AI_Intelligent_Search_Recency_Weight', i18nDescription: 'AI_Intelligent_Search_Recency_Weight_Description', enterprise: true, diff --git a/docs/features/ai-search-hybrid.md b/docs/features/ai-search-hybrid.md index 82a7cac253e2e..4eac6d58c445c 100644 --- a/docs/features/ai-search-hybrid.md +++ b/docs/features/ai-search-hybrid.md @@ -59,7 +59,8 @@ deterministic tie-break, with a semantic preference on exact ties. For nonempty results the weighted score lies in `(0, 1/61]`. At weight 50, it is half the usual unweighted two-list RRF sum; this constant scaling leaves both ordering and multiplicative recency reranking unchanged. For example, semantic rank 3 plus keyword rank 1 gives -`0.5/63 + 0.5/61 = 0.0161332`, ahead of a semantic-only rank-1 hit at `0.5/61 = 0.0081967`. +`0.5/63 + 0.5/61 = 0.0161332`, ahead of a candidate found by the semantic branch alone at rank 1 +(`0.5/61 = 0.0081967`). Both figures are hybrid-mode scores; single-retriever mode is covered below. Single-retriever modes and failure fallback use `1/(60 + rank)`. Rescaling the surviving branch to weight 1 preserves its ordering, including after the multiplicative recency boost. RRF scores are diff --git a/packages/core-services/src/index.ts b/packages/core-services/src/index.ts index 9e9fc7a115abc..bbe4af585ee2e 100644 --- a/packages/core-services/src/index.ts +++ b/packages/core-services/src/index.ts @@ -7,6 +7,7 @@ import type { AISearchModelOption, AISearchResult, AISearchStatus, + AISearchType, } from './types/IAISearchService'; import type { IAbacService } from './types/IAbacService'; import type { IAccount, ILoginResult } from './types/IAccount'; @@ -164,6 +165,7 @@ export type { AISearchModelOption, AISearchResult, AISearchStatus, + AISearchType, ICallHistoryService, IOmnichannelTranscriptService, IQueueWorkerService,