diff --git a/src/api/autocomplete/bench/prefix-bench.ts b/src/api/autocomplete/bench/prefix-bench.ts new file mode 100644 index 0000000..159b40e --- /dev/null +++ b/src/api/autocomplete/bench/prefix-bench.ts @@ -0,0 +1,111 @@ +/* eslint-disable no-console -- CLI benchmark script; console output is the deliverable */ +/** + * Benchmark: prefix autocomplete via Fuse.js scan vs PrefixIndex trie. + * + * Run: npm run bench:prefix (tsx bench/prefix-bench.ts) + * + * Compares three ways of answering a short prefix query over N items: + * 1. Fuse.js as shipped - `^q` WITHOUT useExtendedSearch (broken: the + * caret is fuzzy-matched as a literal character) + * 2. Fuse.js extended - `^q` WITH useExtendedSearch: true (correct, + * but still scans every indexed string per query) + * 3. PrefixIndex trie - O(|prefix| + k) lookup + */ + +import Fuse from "fuse.js"; +import { PrefixIndex } from "../src/prefix-index"; +import { AutocompleteItem } from "../src/types"; + +const N = Number(process.env.BENCH_N || 100_000); +const QUERY_ROUNDS = Number(process.env.BENCH_ROUNDS || 200); +const LIMIT = 10; + +const ADJ = ["fast", "smart", "cloud", "micro", "hyper", "quantum", "neural", "atomic", "green", "solid"]; +const NOUN = ["widget", "gadget", "service", "engine", "parser", "router", "cache", "queue", "stream", "index"]; +const SUF = ["pro", "lite", "max", "core", "kit", "hub", "lab", "box", "net", "base"]; + +function makeItems(n: number): AutocompleteItem[] { + const items: AutocompleteItem[] = []; + for (let i = 0; i < n; i++) { + const a = ADJ[i % ADJ.length]; + const b = NOUN[Math.floor(i / ADJ.length) % NOUN.length]; + const c = SUF[Math.floor(i / (ADJ.length * NOUN.length)) % SUF.length]; + items.push({ + id: `item-${i}`, + title: `${a} ${b} ${c} ${i}`, + description: `The ${a} ${b} for serious ${c} users`, + category: NOUN[i % NOUN.length], + tags: [a, b], + createdAt: new Date(), + updatedAt: new Date(), + }); + } + return items; +} + +// Same 1-2 char prefixes the engine's prefix strategy handles +const QUERIES = ["fa", "sm", "cl", "mi", "hy", "qu", "ne", "at", "gr", "so", "w", "g", "s", "e", "p"]; + +function bench(label: string, fn: () => void, rounds: number): number { + // warmup + for (let i = 0; i < Math.min(10, rounds); i++) fn(); + const t0 = process.hrtime.bigint(); + for (let i = 0; i < rounds; i++) fn(); + const t1 = process.hrtime.bigint(); + const perOpMs = Number(t1 - t0) / 1e6 / rounds; + console.log(`${label.padEnd(34)} ${perOpMs.toFixed(4)} ms/query`); + return perOpMs; +} + +function main(): void { + console.log(`\n=== Prefix autocomplete benchmark: N=${N} items, ${QUERY_ROUNDS} query rounds, limit=${LIMIT} ===\n`); + const items = makeItems(N); + const fuseKeys = [ + { name: "title", weight: 0.7 }, + { name: "description", weight: 0.3 }, + { name: "tags", weight: 0.2 }, + ]; + + let t = Date.now(); + const fusePlain = new Fuse(items, { keys: fuseKeys, threshold: 0.3, includeScore: true }); + console.log(`Fuse (default) index build: ${Date.now() - t} ms`); + + t = Date.now(); + const fuseExt = new Fuse(items, { keys: fuseKeys, threshold: 0.3, includeScore: true, useExtendedSearch: true }); + console.log(`Fuse (extended) index build: ${Date.now() - t} ms`); + + t = Date.now(); + const trie = new PrefixIndex(); + trie.build(items); + console.log(`PrefixIndex trie build: ${Date.now() - t} ms (${trie.tokens} tokens)\n`); + + let qi = 0; + const nextQ = (): string => QUERIES[qi++ % QUERIES.length]; + + const plainMs = bench("Fuse as shipped (`^q`, no ext):", () => { + fusePlain.search(`^${nextQ()}`, { limit: LIMIT }); + }, QUERY_ROUNDS); + + qi = 0; + const extMs = bench("Fuse extended (`^q`, correct):", () => { + fuseExt.search(`^${nextQ()}`, { limit: LIMIT }); + }, QUERY_ROUNDS); + + qi = 0; + const trieMs = bench("PrefixIndex trie:", () => { + trie.search(nextQ(), LIMIT); + }, QUERY_ROUNDS); + + console.log(`\nSpeedup vs Fuse-as-shipped: ${(plainMs / trieMs).toFixed(1)}x`); + console.log(`Speedup vs Fuse-extended: ${(extMs / trieMs).toFixed(1)}x\n`); + + // Correctness spot check: trie results actually start with the prefix + const sample = trie.search("qu", LIMIT); + const ok = sample.length > 0 && sample.every((r) => + r.title.split(/\s+/).some((w) => w.startsWith("qu")) || r.tags.some((tg) => tg.startsWith("qu")) + ); + console.log(`Trie correctness spot check ("qu" -> ${sample.length} results, all prefixed): ${ok ? "PASS" : "FAIL"}`); + if (!ok) process.exit(1); +} + +main(); diff --git a/src/api/autocomplete/package.json b/src/api/autocomplete/package.json index d2b1fc9..dd6a343 100644 --- a/src/api/autocomplete/package.json +++ b/src/api/autocomplete/package.json @@ -20,21 +20,20 @@ "lint:fix": "eslint 'src/**/*.ts' --fix && npm run format", "format": "prettier --write 'src/**/*.{ts,js,json}' '*.md'", "format:check": "prettier --check 'src/**/*.{ts,js,json}' '*.md'", - "openapi": "tsx src/generate-openapi.ts" + "openapi": "tsx src/generate-openapi.ts", + "bench:prefix": "tsx bench/prefix-bench.ts" }, "dependencies": { "@fastify/cors": "^11.2.0", "@fastify/static": "^9.0.0", + "@fastify/swagger": "^9.7.0", "fastify": "^5.7.4", "fuse.js": "^7.0.0", - "lodash": "^4.17.21", - "lodash.debounce": "^4.0.8", "redis": "^4.6.10", "zod": "^3.23.8" }, "devDependencies": { "@types/jest": "^29.5.5", - "@types/lodash.debounce": "^4.0.7", "@types/node": "^20.8.0", "concurrently": "^8.2.2", "jest": "^29.7.0", diff --git a/src/api/autocomplete/src/autocomplete-service.ts b/src/api/autocomplete/src/autocomplete-service.ts index 1721793..79691eb 100644 --- a/src/api/autocomplete/src/autocomplete-service.ts +++ b/src/api/autocomplete/src/autocomplete-service.ts @@ -16,7 +16,6 @@ import { SearchEngine } from "./search-engine"; import { CacheManager, createCacheProvider } from "./cache-manager"; import { DataSourceManager, createDataSource } from "./data-source"; -import debounce from "lodash.debounce"; import { AutocompleteItem, AutocompleteRequest, @@ -35,8 +34,21 @@ export class AutocompleteService { private lastIndexRebuild = new Date(); private indexRebuildTimer?: NodeJS.Timeout; - // Debounced search function for performance - private debouncedSearch: (request: AutocompleteRequest) => Promise; + /** + * Single-flight map: identical concurrent requests share one in-flight + * search instead of each hitting the engine (classic cache-stampede guard). + * + * This replaces the previous lodash.debounce wrapper, which was a genuine + * correctness bug on a server: per lodash's documented contract, + * "subsequent calls to the debounced function return the result of the + * LAST func invocation" - and undefined before the first invocation ever + * runs. Debouncing the shared request path therefore (a) returned + * `undefined` responses during the wait window and (b) leaked one user's + * search response to a different user whose request arrived in the same + * window. Debounce belongs on the client keystroke, never on the server + * request path. + */ + private inflight = new Map>(); constructor(config: AutocompleteConfig) { this.config = config; @@ -53,15 +65,6 @@ export class AutocompleteService { // Initialize data source manager this.dataSourceManager = new DataSourceManager(); - // Create debounced search function - const debouncedFn = debounce(this.performSearch.bind(this), config.api.debounceMs || 300); - - // Wrapper to ensure we always return a Promise - this.debouncedSearch = async (request: AutocompleteRequest): Promise => { - const result = await debouncedFn(request); - return result as AutocompleteResponse; - }; - console.warn("🚀 AutocompleteService initialized"); } @@ -105,12 +108,21 @@ export class AutocompleteService { // Validate request this.validateRequest(request); - // Use debounced search for better performance - if (this.config.api.debounceMs > 0) { - return this.debouncedSearch(request); - } else { - return this.performSearch(request); + // Single-flight: coalesce identical concurrent requests onto one search. + // Distinct requests always run independently and each caller always + // receives the response for ITS OWN request. (config.api.debounceMs is + // intentionally ignored on the server path - see the `inflight` docs.) + const key = this.cacheManager.generateCacheKey(request); + const existing = this.inflight.get(key); + if (existing) { + return existing; } + + const pending = this.performSearch(request).finally(() => { + this.inflight.delete(key); + }); + this.inflight.set(key, pending); + return pending; } /** @@ -324,6 +336,9 @@ export class AutocompleteService { } }, this.config.index.rebuildInterval); + // Background maintenance must not keep the Node.js process alive + this.indexRebuildTimer.unref?.(); + console.warn(`⏰ Scheduled index rebuild every ${this.config.index.rebuildInterval}ms`); } @@ -435,17 +450,8 @@ export class AutocompleteService { this.searchEngine.updateConfig(newConfig.search); } - // Recreate debounced search if debounce time changed - if (newConfig.api?.debounceMs !== undefined) { - const debouncedFn = debounce(this.performSearch.bind(this), newConfig.api.debounceMs); - - this.debouncedSearch = async ( - request: AutocompleteRequest - ): Promise => { - const result = await debouncedFn(request); - return result as AutocompleteResponse; - }; - } + // Note: api.debounceMs is accepted for backward compatibility but is not + // applied on the server request path (see `inflight` docs above). console.warn("⚙️ Service configuration updated"); } diff --git a/src/api/autocomplete/src/autocomplete.test.ts b/src/api/autocomplete/src/autocomplete.test.ts index 3694970..99743c8 100644 --- a/src/api/autocomplete/src/autocomplete.test.ts +++ b/src/api/autocomplete/src/autocomplete.test.ts @@ -13,7 +13,7 @@ import { AutocompleteService } from "./autocomplete-service"; import { SearchEngine } from "./search-engine"; import { CacheManager, MemoryCacheProvider } from "./cache-manager"; import { StaticDataSource, DataSourceManager } from "./data-source"; -import { AutocompleteItem, AutocompleteConfig, DataSource, AutocompleteRequest } from "./types.js"; +import { AutocompleteItem, AutocompleteConfig, DataSource, AutocompleteRequest, AutocompleteResponse } from "./types.js"; const consoleLogSpy = jest.spyOn(console, "log").mockImplementation(() => {}); const consoleWarnSpy = jest.spyOn(console, "warn").mockImplementation(() => {}); @@ -679,3 +679,297 @@ describe("Integration Tests", () => { }); }); }); + +describe("Uplift regression tests", () => { + describe("SearchEngine: prefix strategy uses a real prefix index", () => { + let engine: SearchEngine; + + beforeEach(() => { + engine = new SearchEngine(defaultConfig.search); + engine.buildIndex(sampleItems); + }); + + it("short queries return items whose tokens start with the prefix", async () => { + // Queries of length <= 2 route to the prefix strategy. The old code + // passed "^ja" to Fuse without useExtendedSearch, so the caret was + // matched as a literal character. + const response = await engine.search({ query: "ja", limit: 10 }); + + expect(response.metadata.searchType).toBe("prefix"); + expect(response.results.length).toBeGreaterThan(0); + expect( + response.results.every( + (r) => + r.item.title.toLowerCase().startsWith("ja") || + r.item.title + .toLowerCase() + .split(/\s+/) + .some((w) => w.startsWith("ja")) || + r.item.tags.some((t) => t.toLowerCase().startsWith("ja")) + ) + ).toBe(true); + }); + + it("prefix matches work through tags", async () => { + const response = await engine.search({ query: "ui", limit: 10 }); + + expect(response.results.some((r) => r.item.title === "React")).toBe(true); + }); + }); + + describe("SearchEngine: exact strategy is actually exact", () => { + let engine: SearchEngine; + + beforeEach(() => { + engine = new SearchEngine(defaultConfig.search); + engine.buildIndex(sampleItems); + }); + + it("quoted queries only match items containing the exact phrase", async () => { + const response = await engine.search({ query: '"node.js"', limit: 10 }); + + expect(response.metadata.searchType).toBe("exact"); + expect(response.results.length).toBeGreaterThan(0); + expect( + response.results.every((r) => + [r.item.title, r.item.description || "", ...r.item.tags] + .join(" ") + .toLowerCase() + .includes("node.js") + ) + ).toBe(true); + }); + + it("quoted nonsense phrases match nothing", async () => { + const response = await engine.search({ query: '"zzz not a phrase"', limit: 10 }); + expect(response.results).toHaveLength(0); + }); + }); + + describe("SearchEngine: threshold 0 is honored (falsy-zero bug)", () => { + let engine: SearchEngine; + + beforeEach(() => { + engine = new SearchEngine(defaultConfig.search); + engine.buildIndex(sampleItems); + }); + + it("threshold 0 excludes fuzzy (imperfect) matches", async () => { + // "Javscript" is a typo; every match has score > 0. With threshold 0 + // the old code silently replaced 0 with the 0.3 default and returned + // fuzzy matches anyway. + const response = await engine.search({ query: "Javscript", limit: 10, threshold: 0 }); + + expect(response.results).toHaveLength(0); + }); + }); + + describe("SearchEngine: filters apply before the limit", () => { + it("category filter finds items past the pre-filter cutoff", async () => { + const engine = new SearchEngine(defaultConfig.search); + + // 20 category-A items indexed first, 5 category-B items last. The old + // code fetched only `limit` candidates and THEN filtered, so category B + // + a small limit returned zero results despite 5 matching items. + const bulk: AutocompleteItem[] = []; + for (let i = 0; i < 20; i++) { + bulk.push({ + id: `a-${i}`, + title: `widget alpha ${i}`, + category: "CatA", + tags: ["widget"], + createdAt: new Date(), + updatedAt: new Date(), + }); + } + for (let i = 0; i < 5; i++) { + bulk.push({ + id: `b-${i}`, + title: `widget beta ${i}`, + category: "CatB", + tags: ["widget"], + createdAt: new Date(), + updatedAt: new Date(), + }); + } + engine.buildIndex(bulk); + + // 2-char query -> deterministic prefix strategy + const response = await engine.search({ query: "wi", limit: 3, category: "CatB" }); + + expect(response.results.length).toBeGreaterThan(0); + expect(response.results.every((r) => r.item.category === "CatB")).toBe(true); + expect(response.results.length).toBeLessThanOrEqual(3); + }); + }); + + describe("SearchEngine: highlighting escapes HTML (stored XSS)", () => { + it("item data containing markup is escaped in highlighted fields", async () => { + const engine = new SearchEngine(defaultConfig.search); + engine.buildIndex([ + { + id: "xss-1", + title: ' Widget', + description: ' a widget', + category: "Test", + tags: ["widget"], + createdAt: new Date(), + updatedAt: new Date(), + }, + ]); + + const response = await engine.search({ query: "Widget", limit: 5 }); + + expect(response.results.length).toBeGreaterThan(0); + const { highlightedTitle, highlightedDescription } = response.results[0]; + + expect(highlightedTitle).not.toContain(""); + // tags themselves must survive + expect(`${highlightedTitle}${highlightedDescription}`).toContain(""); + }); + }); + + describe("SearchEngine: popular query average is a true running mean", () => { + it("avgExecutionTime equals the arithmetic mean of recorded times", async () => { + const engine = new SearchEngine(defaultConfig.search); + engine.buildIndex(sampleItems); + + await engine.search({ query: "JavaScript", limit: 5 }); + await engine.search({ query: "JavaScript", limit: 5 }); + await engine.search({ query: "JavaScript", limit: 5 }); + + const analytics = engine.getAnalytics(); + const times = analytics.recentSearches + .filter((s) => s.query === "javascript") + .map((s) => s.executionTime); + const mean = times.reduce((a, b) => a + b, 0) / times.length; + + const popular = analytics.indexStats.popularQueries.find((p) => p.query === "javascript"); + expect(popular).toBeDefined(); + expect(popular!.count).toBe(3); + expect(popular!.avgExecutionTime).toBeCloseTo(mean, 6); + }); + }); + + describe("CacheManager: cache key covers all response-affecting params", () => { + let cacheManager: CacheManager; + + const mockResponse = (query: string): AutocompleteResponse => ({ + query, + results: [], + totalCount: 0, + executionTime: 1, + metadata: { searchType: "fuzzy" as const, cacheHit: false, indexSize: 5 }, + }); + + beforeEach(() => { + cacheManager = new CacheManager(new MemoryCacheProvider(50), defaultConfig.cache); + }); + + it("different categories never share a cache entry (poisoning regression)", async () => { + await cacheManager.set({ query: "x", category: "books" }, mockResponse("x")); + + const other = await cacheManager.get({ query: "x", category: "movies" }); + const same = await cacheManager.get({ query: "x", category: "books" }); + + expect(other).toBeNull(); + expect(same).not.toBeNull(); + }); + + it("different tags never share a cache entry", async () => { + await cacheManager.set({ query: "x", tags: ["a"] }, mockResponse("x")); + + expect(await cacheManager.get({ query: "x", tags: ["b"] })).toBeNull(); + expect(await cacheManager.get({ query: "x", tags: ["a"] })).not.toBeNull(); + }); + + it("tag order does not fragment the cache", async () => { + await cacheManager.set({ query: "x", tags: ["a", "b"] }, mockResponse("x")); + + expect(await cacheManager.get({ query: "x", tags: ["b", "a"] })).not.toBeNull(); + }); + + it("query case/whitespace is normalized (strict assertion, not toBeDefined)", async () => { + await cacheManager.set({ query: "Test", limit: 5 }, mockResponse("test")); + + expect(await cacheManager.get({ query: " test ", limit: 5 })).not.toBeNull(); + }); + + it("fuzzy default (undefined) and explicit true share an entry", async () => { + await cacheManager.set({ query: "x" }, mockResponse("x")); + + expect(await cacheManager.get({ query: "x", fuzzy: true })).not.toBeNull(); + expect(await cacheManager.get({ query: "x", fuzzy: false })).toBeNull(); + }); + }); + + describe("AutocompleteService: server-side debounce removed", () => { + it("concurrent distinct requests each receive their own response, even with debounceMs configured", async () => { + // With the old lodash.debounce wrapper and debounceMs > 0, concurrent + // callers either received `undefined` (no prior invocation) or the + // LAST caller's response - cross-request response leakage. + const service = new AutocompleteService({ + ...defaultConfig, + api: { ...defaultConfig.api, debounceMs: 300 }, + }); + await service.initialize([ + { + id: "s", + name: "s", + type: "static", + config: { data: sampleItems }, + itemCount: sampleItems.length, + }, + ]); + + try { + const [a, b] = await Promise.all([ + service.search({ query: "JavaScript", limit: 5 }), + service.search({ query: "Python", limit: 5 }), + ]); + + expect(a).toBeDefined(); + expect(b).toBeDefined(); + expect(a.query).toBe("javascript"); + expect(b.query).toBe("python"); + expect(a.results.some((r) => r.item.title === "JavaScript")).toBe(true); + expect(b.results.some((r) => r.item.title === "Python")).toBe(true); + } finally { + await service.shutdown(); + } + }); + + it("identical concurrent requests are single-flighted onto one engine search", async () => { + const service = new AutocompleteService({ + ...defaultConfig, + cache: { ...defaultConfig.cache, enabled: false }, // isolate single-flight from cache + }); + await service.initialize([ + { + id: "s", + name: "s", + type: "static", + config: { data: sampleItems }, + itemCount: sampleItems.length, + }, + ]); + + try { + const engine = (service as unknown as { searchEngine: SearchEngine }).searchEngine; + const spy = jest.spyOn(engine, "search"); + + const responses = await Promise.all( + Array.from({ length: 5 }, () => service.search({ query: "JavaScript", limit: 5 })) + ); + + expect(spy).toHaveBeenCalledTimes(1); + expect(responses).toHaveLength(5); + expect(responses.every((r) => r.results.length > 0)).toBe(true); + } finally { + await service.shutdown(); + } + }); + }); +}); diff --git a/src/api/autocomplete/src/cache-manager.ts b/src/api/autocomplete/src/cache-manager.ts index 6a4bfb5..bbf9110 100644 --- a/src/api/autocomplete/src/cache-manager.ts +++ b/src/api/autocomplete/src/cache-manager.ts @@ -374,11 +374,23 @@ export class CacheManager { * Ensures consistent key generation for identical requests */ generateCacheKey(request: AutocompleteRequest): string { + // The key MUST cover every request parameter that changes the response. + // The previous key omitted `category` and `tags`, so a cached result for + // ?q=x&category=books was served verbatim to ?q=x&category=movies - + // cross-filter cache poisoning. Query/category/tags are normalized the + // same way the search engine normalizes them, so equivalent requests + // ("Test" vs "test") share one entry instead of duplicating cache slots. const parts = [ - request.query, + request.query.trim().toLowerCase(), request.limit?.toString() || "default", - request.fuzzy?.toString() || "false", - request.threshold?.toString() || "default", + // Engine semantics: fuzzy defaults to TRUE (fuzzy !== false). The old + // key stringified undefined as "false" - the opposite of reality. + (request.fuzzy !== false).toString(), + request.threshold !== undefined ? request.threshold.toString() : "default", + request.category?.trim().toLowerCase() || "any", + request.tags && request.tags.length > 0 + ? [...request.tags].map((t) => t.trim().toLowerCase()).sort().join(",") + : "any", request.fields?.join(",") || "all", ]; diff --git a/src/api/autocomplete/src/prefix-index.test.ts b/src/api/autocomplete/src/prefix-index.test.ts new file mode 100644 index 0000000..d526ccb --- /dev/null +++ b/src/api/autocomplete/src/prefix-index.test.ts @@ -0,0 +1,152 @@ +/** + * Unit tests for PrefixIndex (trie-backed prefix autocomplete). + */ + +import { PrefixIndex } from "./prefix-index"; +import { AutocompleteItem } from "./types.js"; + +function item(id: string, title: string, tags: string[] = [], category = "Test"): AutocompleteItem { + return { + id, + title, + category, + tags, + createdAt: new Date("2023-01-01"), + updatedAt: new Date("2023-01-01"), + }; +} + +describe("PrefixIndex", () => { + let index: PrefixIndex; + + beforeEach(() => { + index = new PrefixIndex(); + }); + + it("matches items by title prefix", () => { + index.build([item("1", "JavaScript"), item("2", "Java"), item("3", "Python")]); + + const results = index.search("jav", 10); + const titles = results.map((r) => r.title).sort(); + + expect(titles).toEqual(["Java", "JavaScript"]); + }); + + it("matches items by individual title-word prefix", () => { + index.build([item("1", "Node.js Runtime"), item("2", "Deno Runtime"), item("3", "Python")]); + + const results = index.search("run", 10); + const ids = results.map((r) => r.id).sort(); + + expect(ids).toEqual(["1", "2"]); + }); + + it("matches items by tag prefix", () => { + index.build([ + item("1", "React", ["frontend", "ui"]), + item("2", "Express", ["backend"]), + ]); + + const results = index.search("ui", 10); + + expect(results.map((r) => r.id)).toEqual(["1"]); + }); + + it("is case-insensitive on both sides", () => { + index.build([item("1", "TypeScript")]); + + expect(index.search("TYPE", 10)).toHaveLength(1); + expect(index.search("tYpEsC", 10)).toHaveLength(1); + }); + + it("returns distinct items even when multiple tokens match", () => { + // "java" matches the full title AND the first word of "Java Virtual Machine" + index.build([item("1", "Java Virtual Machine", ["java"])]); + + const results = index.search("java", 10); + + expect(results).toHaveLength(1); + expect(results[0].id).toBe("1"); + }); + + it("respects the limit", () => { + const items = Array.from({ length: 50 }, (_, i) => item(`id-${i}`, `alpha${i}`)); + index.build(items); + + expect(index.search("alpha", 7)).toHaveLength(7); + expect(index.search("alpha", 0)).toHaveLength(0); + }); + + it("ranks exact token matches before longer completions", () => { + index.build([item("long", "golang"), item("exact", "go")]); + + const results = index.search("go", 10); + + expect(results[0].id).toBe("exact"); + expect(results[1].id).toBe("long"); + }); + + it("handles unicode queries without splitting characters", () => { + index.build([item("1", "Héllo Wörld"), item("2", "🚀 Rocket")]); + + expect(index.search("hé", 10).map((r) => r.id)).toEqual(["1"]); + expect(index.search("wö", 10).map((r) => r.id)).toEqual(["1"]); + expect(index.search("🚀", 10).map((r) => r.id)).toEqual(["2"]); + }); + + it("does NOT treat the query as a fuzzy pattern or regex", () => { + index.build([item("1", "abc"), item("2", "^ab")]); + + // "^a" must only match the literal title "^ab", never "abc" + expect(index.search("^a", 10).map((r) => r.id)).toEqual(["2"]); + // "a." must not regex-match "ab"/"abc" + expect(index.search("a.", 10)).toHaveLength(0); + }); + + it("returns empty results for empty or unmatched prefixes", () => { + index.build([item("1", "JavaScript")]); + + expect(index.search("", 10)).toEqual([]); + expect(index.search(" ", 10)).toEqual([]); + expect(index.search("zzz", 10)).toEqual([]); + }); + + it("supports has() prefix membership checks", () => { + index.build([item("1", "JavaScript", ["web"])]); + + expect(index.has("java")).toBe(true); + expect(index.has("we")).toBe(true); + expect(index.has("xyz")).toBe(false); + expect(index.has("")).toBe(false); + }); + + it("rebuild replaces previous contents entirely", () => { + index.build([item("1", "OldEntry")]); + index.build([item("2", "NewEntry")]); + + expect(index.search("old", 10)).toHaveLength(0); + expect(index.search("new", 10).map((r) => r.id)).toEqual(["2"]); + expect(index.size).toBe(1); + }); + + it("reports token statistics", () => { + index.build([item("1", "Node.js Runtime", ["backend"])]); + + // tokens: "node.js runtime", "node.js", "runtime", "backend" + expect(index.tokens).toBe(4); + expect(index.size).toBe(1); + }); + + it("stays correct on a 10k-item dataset", () => { + const vocab = ["alpha", "beta", "gamma", "delta", "omega", "sigma"]; + const items = Array.from({ length: 10000 }, (_, i) => + item(`id-${i}`, `${vocab[i % vocab.length]} tool ${i}`) + ); + index.build(items); + + const results = index.search("gam", 25); + + expect(results).toHaveLength(25); + expect(results.every((r) => r.title.startsWith("gamma"))).toBe(true); + }); +}); diff --git a/src/api/autocomplete/src/prefix-index.ts b/src/api/autocomplete/src/prefix-index.ts new file mode 100644 index 0000000..102b709 --- /dev/null +++ b/src/api/autocomplete/src/prefix-index.ts @@ -0,0 +1,158 @@ +/** + * PrefixIndex - character trie for true prefix autocomplete. + * + * Why this exists: the previous "prefix" strategy passed `^query` to Fuse.js, + * but the `^` prefix operator only works when `useExtendedSearch: true` is set + * (https://www.fusejs.io/extended-search.html). It never was, so the caret was + * fuzzy-matched as a literal character. Even with the operator enabled, Fuse + * scans every indexed string per query (bitap), i.e. O(n * m). A trie answers + * prefix queries in O(|prefix| + k) - independent of dataset size. + * + * Indexed tokens per item: full lowercase title, each whitespace-separated + * title word, and each tag. Lookup returns distinct items, exact-token + * matches first, then lexicographic DFS order (deterministic). + * + * Unicode: iteration uses for..of (code points), so surrogate pairs are not + * split mid-character. + */ + +import { AutocompleteItem } from "./types.js"; + +interface TrieNode { + children: Map; + /** Indices into the items array for items with an indexed token ending here. */ + itemIndices: number[] | null; +} + +function newNode(): TrieNode { + return { children: new Map(), itemIndices: null }; +} + +export class PrefixIndex { + private root: TrieNode = newNode(); + private items: AutocompleteItem[] = []; + private tokenCount = 0; + + /** Build (or rebuild) the trie from a list of items. */ + build(items: AutocompleteItem[]): void { + this.root = newNode(); + this.items = items; + this.tokenCount = 0; + + for (let idx = 0; idx < items.length; idx++) { + const item = items[idx]; + const tokens = new Set(); + + const title = (item.title || "").toLowerCase().trim(); + if (title) { + tokens.add(title); + for (const word of title.split(/\s+/)) { + if (word) tokens.add(word); + } + } + for (const tag of item.tags || []) { + const t = (tag || "").toLowerCase().trim(); + if (t) tokens.add(t); + } + + for (const token of tokens) { + this.insert(token, idx); + } + } + } + + private insert(token: string, itemIdx: number): void { + let node = this.root; + for (const ch of token) { + let next = node.children.get(ch); + if (!next) { + next = newNode(); + node.children.set(ch, next); + } + node = next; + } + if (!node.itemIndices) { + node.itemIndices = []; + this.tokenCount++; + } + // Distinct tokens per item guarantee no duplicate push for the same token; + // guard against repeated calls anyway. + if (node.itemIndices[node.itemIndices.length - 1] !== itemIdx) { + node.itemIndices.push(itemIdx); + } + } + + /** + * Return up to `limit` distinct items having any indexed token that starts + * with `prefix`. Case-insensitive. O(|prefix| + limit * branch) time. + */ + search(prefix: string, limit: number = 10): AutocompleteItem[] { + const p = prefix.toLowerCase().trim(); + if (!p || limit <= 0) return []; + + // Walk down to the node for the prefix. + let node: TrieNode = this.root; + for (const ch of p) { + const next = node.children.get(ch); + if (!next) return []; + node = next; + } + + const seen = new Set(); + const out: AutocompleteItem[] = []; + + const collect = (indices: number[] | null): void => { + if (!indices) return; + for (const i of indices) { + if (seen.has(i)) continue; + seen.add(i); + out.push(this.items[i]); + if (out.length >= limit) return; + } + }; + + // Exact token matches rank first. + collect(node.itemIndices); + if (out.length >= limit) return out; + + // Then lexicographic depth-first over the subtree. + const stack: TrieNode[] = []; + const pushChildren = (n: TrieNode): void => { + const keys = [...n.children.keys()].sort().reverse(); // reverse so pop() yields ascending + for (const k of keys) stack.push(n.children.get(k)!); + }; + pushChildren(node); + + while (stack.length > 0 && out.length < limit) { + const n = stack.pop()!; + collect(n.itemIndices); + if (out.length >= limit) break; + pushChildren(n); + } + + return out; + } + + /** Whether any indexed token starts with the given prefix. */ + has(prefix: string): boolean { + const p = prefix.toLowerCase().trim(); + if (!p) return false; + let node: TrieNode = this.root; + for (const ch of p) { + const next = node.children.get(ch); + if (!next) return false; + node = next; + } + return true; + } + + /** Number of indexed items. */ + get size(): number { + return this.items.length; + } + + /** Number of distinct indexed tokens. */ + get tokens(): number { + return this.tokenCount; + } +} diff --git a/src/api/autocomplete/src/search-engine.ts b/src/api/autocomplete/src/search-engine.ts index bc1f574..39c5021 100644 --- a/src/api/autocomplete/src/search-engine.ts +++ b/src/api/autocomplete/src/search-engine.ts @@ -25,6 +25,7 @@ import { IndexStats, } from "./types.js"; import { SearchLimit } from "./constants"; +import { PrefixIndex } from "./prefix-index"; /** Result item shape from Fuse.search() - mirrors Fuse.js FuseResult for ESM compatibility */ interface FuseResultItem { @@ -42,6 +43,7 @@ type FuseResultMatchItem = NonNullable[number]; export class SearchEngine { private fuseIndex: Fuse | null = null; + private prefixIndex: PrefixIndex = new PrefixIndex(); private items: AutocompleteItem[] = []; private config: SearchConfig; private analytics: SearchAnalytics[] = []; @@ -105,6 +107,9 @@ export class SearchEngine { ignoreFieldNorm: this.config.ignoreFieldNorm, }); + // Build the trie for O(|prefix| + k) prefix lookups. + this.prefixIndex.build(this.items); + const buildTime = Date.now() - startTime; // Update index statistics @@ -140,8 +145,14 @@ export class SearchEngine { // Perform the actual search const searchResults = await this.performSearch(sanitizedRequest, searchStrategy); - // Apply post-processing filters - const filteredResults = this.applyFilters(searchResults, sanitizedRequest); + // Apply post-processing filters, THEN the limit. Limiting before + // filtering (the old order) silently dropped matching items: a category + // filter could return 0 results even though matching items existed just + // past the pre-filter cutoff. + const filteredResults = this.applyFilters(searchResults, sanitizedRequest).slice( + 0, + sanitizedRequest.limit || SearchLimit.DEFAULT_LIMIT + ); // Generate query suggestions if results are limited const suggestions = await this.generateSuggestions(sanitizedRequest, filteredResults); @@ -187,7 +198,9 @@ export class SearchEngine { category: request.category?.trim(), tags: request.tags?.map((tag) => tag.trim().toLowerCase()), fuzzy: request.fuzzy !== false, // Default to true - threshold: Math.max(0, Math.min(1, request.threshold || this.config.threshold)), + // ?? (not ||): threshold 0 means "exact matches only" and must survive. + // `request.threshold || default` silently replaced 0 with the default. + threshold: Math.max(0, Math.min(1, request.threshold ?? this.config.threshold)), }; } @@ -223,37 +236,55 @@ export class SearchEngine { request: AutocompleteRequest, strategy: "exact" | "fuzzy" | "prefix" ): Promise { - let fuseResults: FuseResultItem[] = []; const limit = request.limit || SearchLimit.DEFAULT_LIMIT; + // If category/tag/threshold filters will run AFTER this retrieval, we must + // retrieve more than `limit` candidates or the filters can starve + // legitimate matches that sit past the cutoff. + // (threshold is deliberately excluded: results are score-sorted, so a + // score cutoff applied to the top-k window can never starve matches) + const hasPostFilters = !!(request.category || (request.tags && request.tags.length > 0)); + const retrieveLimit = hasPostFilters ? this.items.length : limit; + + let results: SearchResult[]; + switch (strategy) { - case "exact": - // For exact search, use a very low threshold - fuseResults = this.fuseIndex!.search(request.query, { - limit: limit, - }) as FuseResultItem[]; + case "exact": { + // Previously "exact" ran the same fuzzy Fuse search as everything + // else. Now it is an actual exact phrase match (quotes stripped) + // over title, description, and tags. + const phrase = request.query.replace(/["']/g, "").trim(); + results = []; + for (const item of this.items) { + const haystacks = [item.title, item.description || "", ...(item.tags || [])]; + if (haystacks.some((h) => h.toLowerCase().includes(phrase))) { + results.push(this.buildDirectResult(item, phrase)); + if (results.length >= retrieveLimit) break; + } + } break; + } - case "prefix": - // For prefix search, look for items that start with the query - fuseResults = this.fuseIndex!.search(`^${request.query}`, { - limit: limit, - }) as FuseResultItem[]; + case "prefix": { + // Trie lookup: O(|prefix| + k). The old code passed `^query` to + // Fuse.js, but the `^` operator requires useExtendedSearch: true + // (never set), so the caret was fuzzy-matched as a literal char. + const items = this.prefixIndex.search(request.query, retrieveLimit); + results = items.map((item) => this.buildDirectResult(item, request.query)); break; + } case "fuzzy": - default: - // Standard fuzzy search - fuseResults = this.fuseIndex!.search(request.query, { - limit: limit, + default: { + const fuseResults = this.fuseIndex!.search(request.query, { + limit: retrieveLimit, }) as FuseResultItem[]; + results = fuseResults.map((fuseResult) => + this.convertFuseResult(fuseResult, request.query) + ); break; + } } - - // Prefer exact title matches when query length >= 4 - const results = fuseResults.map((fuseResult) => - this.convertFuseResult(fuseResult, request.query) - ); if (request.query.length >= 4) { results.sort((a, b) => { const aTitleExact = a.item.title.toLowerCase() === request.query.toLowerCase() ? -1 : 0; @@ -304,33 +335,79 @@ export class SearchEngine { } /** - * Apply highlighting to matched text + * Escape HTML special characters. Highlighted fields are HTML fragments + * (they contain tags), so item data MUST be escaped before marks + * are inserted or any item whose title/description contains markup becomes + * a stored-XSS vector when the frontend renders the fragment. + */ + private escapeHtml(text: string): string { + return text + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); + } + + /** + * Build a SearchResult directly from an item (prefix/exact strategies), + * highlighting the first case-insensitive occurrence of the query. + */ + private buildDirectResult(item: AutocompleteItem, query: string): SearchResult { + const highlight = (text: string | undefined): string | undefined => { + if (text === undefined) return undefined; + const idx = query ? text.toLowerCase().indexOf(query.toLowerCase()) : -1; + if (idx < 0) return this.escapeHtml(text); + return ( + this.escapeHtml(text.slice(0, idx)) + + `${this.escapeHtml(text.slice(idx, idx + query.length))}` + + this.escapeHtml(text.slice(idx + query.length)) + ); + }; + + return { + item, + score: 0, // Direct (non-fuzzy) matches are perfect matches in Fuse terms + matches: [], + highlightedTitle: highlight(item.title), + highlightedDescription: highlight(item.description), + }; + } + + /** + * Apply highlighting to matched text. + * Escapes HTML (see escapeHtml) and merges overlapping match ranges so the + * output never contains nested/broken tags. */ private highlightMatches(text: string, match?: FuseResultMatchItem): string { if (!match || !match.indices || match.indices.length === 0) { - return text; + return this.escapeHtml(text); } - let highlightedText = ""; - let lastIndex = 0; - - // Sort indices to process them in order - const sortedIndices = [...match.indices].sort( + // Sort, then merge overlapping/adjacent ranges + const sorted = [...match.indices].sort( (a: readonly [number, number], b: readonly [number, number]) => a[0] - b[0] ); + const merged: Array<[number, number]> = []; + for (const [start, end] of sorted) { + const last = merged[merged.length - 1]; + if (last && start <= last[1] + 1) { + last[1] = Math.max(last[1], end); + } else { + merged.push([start, end]); + } + } - for (const [start, end] of sortedIndices) { - // Add text before the match - highlightedText += text.slice(lastIndex, start); - - // Add highlighted match - highlightedText += `${text.slice(start, end + 1)}`; + let highlightedText = ""; + let lastIndex = 0; + for (const [start, end] of merged) { + highlightedText += this.escapeHtml(text.slice(lastIndex, start)); + highlightedText += `${this.escapeHtml(text.slice(start, end + 1))}`; lastIndex = end + 1; } - // Add remaining text - highlightedText += text.slice(lastIndex); + highlightedText += this.escapeHtml(text.slice(lastIndex)); return highlightedText; } @@ -357,8 +434,9 @@ export class SearchEngine { ); } - // Apply score threshold - if (request.threshold) { + // Apply score threshold (!== undefined, not truthiness: threshold 0 + // means "perfect matches only" and was previously skipped entirely) + if (request.threshold !== undefined) { filteredResults = filteredResults.filter( (result) => result.score <= request.threshold! // Lower score = better match in Fuse.js ); @@ -457,9 +535,13 @@ export class SearchEngine { const existing = this.indexStats.popularQueries.find((pq) => pq.query === query); if (existing) { - existing.count++; + // True running mean. The old formula ((avg + latest) / 2) is an + // exponentially-weighted drift, not an average: after N searches the + // first sample's weight is 1/2^(N-1). + const latest = this.analytics[this.analytics.length - 1].executionTime; existing.avgExecutionTime = - (existing.avgExecutionTime + this.analytics[this.analytics.length - 1].executionTime) / 2; + (existing.avgExecutionTime * existing.count + latest) / (existing.count + 1); + existing.count++; } else { this.indexStats.popularQueries.push({ query,