From bada46836a95d5d1d1787b6027585d609e417373 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=9D=E5=84=BF?= <274762368+xcjy8bao@users.noreply.github.com> Date: Wed, 23 Sep 2026 11:15:45 +0800 Subject: [PATCH 1/2] fix: improve semantic search and prepare 1.0.1 release --- CHANGELOG.md | 15 ++++ README.md | 4 +- README.zh-CN.md | 4 +- package.json | 2 +- plugins/sift-light/.claude-plugin/plugin.json | 2 +- plugins/sift-light/.codex-plugin/plugin.json | 2 +- plugins/sift-light/.mcp.json | 2 +- plugins/sift-light/concept-worker.mjs | 51 ++++++++--- plugins/sift-light/kimi.plugin.json | 4 +- plugins/sift-light/omp-extension.mjs | 61 +++++++++---- plugins/sift-light/package.json | 2 +- src/concept-inference.ts | 55 +++++++++--- src/concept-model.ts | 4 +- src/concept-search.ts | 14 ++- src/concept-worker.mjs | 51 ++++++++--- src/concept-worker.ts | 2 +- src/hybrid-search.ts | 13 +++ src/mcp-model-output.ts | 41 +++++++++ src/mcp-server.mjs | 87 +++++++++++++++---- src/model-error.ts | 22 ++--- src/request-contract.ts | 46 ++++++++-- src/semantic-judge.ts | 3 +- 22 files changed, 379 insertions(+), 108 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e69de29b..4bd1dcd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -0,0 +1,15 @@ +# Changelog + +## 1.0.1 — 2026-09-23 + +### Fixed + +- Improve semantic ranking when very short passages compete with more detailed source evidence. Raw cosine similarity remains available alongside the ranking score; results are still relevance candidates, not verified behavior. +- Give `files` requests that mistakenly use a plain `pattern` a complete, scope-preserving `query` recovery request. Reduce repeated request-error text. +- Condense hybrid and Concept diagnostics in model-facing output while keeping coverage, progress, recovery, and semantic-judge status visible. +- Ask the optional Jev semantic judge to classify each candidate by its own excerpt, avoiding a shared classification instruction across candidates. + +### Performance and compatibility + +- Use length-aware inference ordering, batches of four, four ONNX CPU threads, and a shorter token window to reduce cold semantic search time on the tested workloads. The changed windowing invalidates older embedding cache entries once; subsequent searches refill the cache. +- Cold semantic inference can still take tens of seconds and use more than 1 GiB of worker memory. The improvement is workload-dependent and is not a memory cap or a subsecond cold-start guarantee. diff --git a/README.md b/README.md index 1126f5f6..f4fc0c5e 100644 --- a/README.md +++ b/README.md @@ -24,12 +24,14 @@ Looking for an error message, a sentence or a name is like giving a librarian a ### Recover when the remembered wording is not exact -Use `mode: "hybrid"` with one natural-language `query` when a sentence may have been remembered with different wording. Hybrid always runs an exact literal search and the installed local Concept model under one owned request. Exact evidence appears first; semantic candidates are clearly labeled, ranked only by similarity and removed when they overlap an exact match. The initial page shares counts, coverage, source references, one inspection cursor and a compact preview instead of concatenating two complete responses. `conceptLimit` changes only the non-overlapping semantic supplement (default 3, maximum 20); it never displaces literal evidence. The returned matches request opens the same snapshot's complete exact-first pagination without rerunning either search. +Use `mode: "hybrid"` with one natural-language `query` when a sentence may have been remembered with different wording. Hybrid always runs an exact literal search and the installed local Concept model under one owned request. Exact evidence appears first; semantic candidates are clearly labeled as relevance candidates and removed when they overlap an exact match. Semantic candidates retain raw cosine scores and use a bounded short-passage correction for ranking; both scores remain visible in structured details and neither proves runtime behavior. The initial page shares counts, coverage, source references, one inspection cursor and a compact preview instead of concatenating two complete responses. `conceptLimit` changes only the non-overlapping semantic supplement (default 3, maximum 20); it never displaces literal evidence. The returned matches request opens the same snapshot's complete exact-first pagination without rerunning either search. Concept ranking covers every UTF-8 passage admitted by the request's documented source budget; it no longer samples a fixed prefix of the scope. Concept and hybrid searches automatically admit up to 2,000 files and process them sequentially in bounded 200-file batches, then merge every batch into one global ranking and one coverage result. Batches share the request's 32 MiB read budget, so raising the file ceiling does not multiply the content budget. Users do not need to plan or continue batches themselves. `maxFilesToParse` remains an optional advanced hard ceiling when a smaller scope is intentional. Passages that exceed the model token window are ranked through overlapping token-safe windows, so later text is not silently discarded. Offline embeddings are cached by content, model revision and chunking revision in a bounded 512 MiB local cache. Repeated content is reused, changed content misses naturally, and cache write or cleanup failures remain visible in the result. Slow Concept and hybrid requests return within the default five-second wait window with `status: "waiting"` or `"running"`, an `operationId`, progress and an exact `nextRequest` such as `{ "mode": "await", "operationId": "..." }`. Copy that request unchanged: it resumes the same computation and never restarts the query or downgrades to a literal-only result. A final result remains available for stable re-fetch for 10 minutes, with up to 32 terminal results retained per service session, and `mode: "cancel"` stops the owned work and waits for cleanup. Each service session admits at most eight pending operations; an operation has one total deadline controlled by `SIFT_LIGHT_CONCEPT_TIMEOUT_MS` (integer milliseconds from 1000 through 3600000; default 600000) and a 120-second idle continuation lease. A real model, source or resource failure is returned as a failure with its diagnostic. Source generation is re-enumerated and re-verified before publication, so changes refresh the operation and mixed versions are never marked complete. Admission planning counts (`filesEnumerated`, `filesAdmitted`, `filesSkippedEmpty`, `filesUnavailable`, `passagesQueued`, `batchesPlanned`, `batchesCompleted`) stay visible. Empty files are a normal skip and do not mark the result partial. +The first uncached Concept or hybrid search loads the local model and may take tens of seconds and more than 1 GiB of inference-worker memory, depending on the machine and search scope. Cached searches avoid most inference work. These are workload-dependent observations, not a latency or memory guarantee. If the result is waiting during `model-loading`, follow its `nextRequest` to continue the same operation. + ### Give several search conditions together “Find files mentioning both the customer and a refund” works like selecting documents with two labels. “Any of these words will do” works like handing over a shortlist. Multiple conditions can be expressed together to reduce repeated searches. diff --git a/README.zh-CN.md b/README.zh-CN.md index 475c9111..51cd4093 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -24,12 +24,14 @@ ### 记得不完全准确，也能一次找回 -当一句话可能记错了措辞时，可以用一个自然语言 `query` 调用 `mode: "hybrid"`。Hybrid 会在同一个受控请求中始终执行精确字面搜索和已安装的本地 Concept 模型：精确证据固定排在前面，语义候选明确标注且只表示相似度，与精确命中范围重叠的候选会被去重。初始页面共享计数、覆盖状态、来源引用和一个检查游标，以紧凑预览代替拼接两份完整响应。`conceptLimit` 只调整不重叠的语义补充数量（默认 3，最大 20），不会挤占字面证据；返回的 matches 请求从同一个快照开始完整的精确优先分页，不会重新执行任一搜索。 +当一句话可能记错了措辞时，可以用一个自然语言 `query` 调用 `mode: "hybrid"`。Hybrid 会在同一个受控请求中始终执行精确字面搜索和已安装的本地 Concept 模型：精确证据固定排在前面，语义候选明确标注且只表示相关性候选，与精确命中范围重叠的候选会被去重。语义候选保留原始余弦分数，并用有界短段修正分数排序；两种分数均在结构化详情中可见，不能当作运行时证明。初始页面共享计数、覆盖状态、来源引用和一个检查游标，以紧凑预览代替拼接两份完整响应。`conceptLimit` 只调整不重叠的语义补充数量（默认 3，最大 20），不会挤占字面证据；返回的 matches 请求从同一个快照开始完整的精确优先分页，不会重新执行任一搜索。 Concept 排名会覆盖请求所声明源码预算内接纳的全部 UTF-8 段落，不再固定抽取范围开头的一小部分。Concept 和 hybrid 默认会自动接纳最多 2,000 个文件，在内部按每批 200 个文件顺序处理，再合并成一次全局排名和一份覆盖结果。所有批次共享同一个请求的 32 MiB 读取预算，扩大文件上限不会把内容预算成倍放大。用户不需要自己计算或续接批次；只有确实想主动缩小范围时，才需要把 `maxFilesToParse` 作为可选的高级硬上限。超过模型 token 窗口的段落会拆成带重叠、且保证不截断的窗口参与排名，后半段内容不会被静默丢弃。离线 embedding 按内容、模型版本和分段版本缓存在本地，缓存上限为 512 MiB；重复内容直接复用，内容变化自然失效，缓存写入或清理失败会在结果中明确显示。 Concept 或 hybrid 较慢时，会在默认五秒等待窗口内返回 `status: "waiting"` 或 `"running"`、`operationId`、进度和精确的 `nextRequest`，例如 `{ "mode": "await", "operationId": "..." }`。请原样复制这个请求：它会续接同一个计算，不会重启查询，也不会降级成只有字面的结果。最终结果可稳定复取十分钟；每个服务会话最多保留 32 个终态结果。`mode: "cancel"` 会停止自有任务并等待清理完成。每个服务会话最多同时接纳八个 pending operation；单个 operation 使用 `SIFT_LIGHT_CONCEPT_TIMEOUT_MS` 指定一个总执行时限（整数毫秒，1000–3600000，默认 600000），另有 120 秒无人续接租期。真实模型、来源或资源故障会以明确失败返回。发布结果前会重新枚举并校验同一来源 generation；源文件变化会刷新 operation，混合版本不会被标成 complete。接纳计划计数（`filesEnumerated`、`filesAdmitted`、`filesSkippedEmpty`、`filesUnavailable`、`passagesQueued`、`batchesPlanned`、`batchesCompleted`）会在结果里明确显示。空文件属于正常跳过，不会把结果标成 partial。 +首次运行尚未缓存的 Concept 或 hybrid 搜索时需要加载本地模型；耗时可能达到数十秒，推理 worker 内存也可能超过 1 GiB，具体取决于机器和搜索范围。缓存热后可省去大部分推理工作。这只是随负载变化的观察，不是延迟或内存保证。结果在 `model-loading` 阶段等待时，请按返回的 `nextRequest` 续接同一个 operation。 + ### 几个条件，可以一起交代 “找同时提到客户和退款的文件”，就像请管理员挑出同时贴着两张标签的资料；“这几个词任意一个出现都算”，则像列出一张候选清单。可以一次表达多个查找条件，减少反复搜索。 diff --git a/package.json b/package.json index 1f647012..8c289eda 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sift-light", - "version": "1.0.0", + "version": "1.0.1", "description": "Context-efficient local search for files, documents, notes and logs across Pi, OMP and MCP clients", "keywords": [ "ai-agent", diff --git a/plugins/sift-light/.claude-plugin/plugin.json b/plugins/sift-light/.claude-plugin/plugin.json index 9de2cde1..6309f297 100644 --- a/plugins/sift-light/.claude-plugin/plugin.json +++ b/plugins/sift-light/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "sift-light", - "version": "1.0.0", + "version": "1.0.1", "description": "Require sift-light for conventional local searches while keeping development tools available.", "author": { "name": "baoer" diff --git a/plugins/sift-light/.codex-plugin/plugin.json b/plugins/sift-light/.codex-plugin/plugin.json index 25effa5e..3f2a8716 100644 --- a/plugins/sift-light/.codex-plugin/plugin.json +++ b/plugins/sift-light/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "sift-light", - "version": "1.0.0", + "version": "1.0.1", "description": "Require sift-light for conventional local searches while keeping development tools available.", "author": { "name": "baoer" diff --git a/plugins/sift-light/.mcp.json b/plugins/sift-light/.mcp.json index c0f567d4..16aa32da 100644 --- a/plugins/sift-light/.mcp.json +++ b/plugins/sift-light/.mcp.json @@ -5,7 +5,7 @@ "args": [ "--yes", "--package", - "sift-light-runtime@npm:sift-light@1.0.0", + "sift-light-runtime@npm:sift-light@1.0.1", "sift-light-mcp", "--stdio" ], diff --git a/plugins/sift-light/concept-worker.mjs b/plugins/sift-light/concept-worker.mjs index d09872aa..84eabc6a 100755 --- a/plugins/sift-light/concept-worker.mjs +++ b/plugins/sift-light/concept-worker.mjs @@ -27,10 +27,10 @@ class SiftLightError extends Error { var CONCEPT_MODEL = "Xenova/multilingual-e5-small"; var CONCEPT_REVISION = "761b726dd34fb83930e26aab4e9ac3899aa1fa78"; var MAX_CONCEPT_CHARS = 1000; -var CONCEPT_MODEL_TOKENS = 512; +var CONCEPT_MODEL_TOKENS = 256; var CONCEPT_WINDOW_OVERLAP_TOKENS = 64; var CONCEPT_EMBEDDING_DIMENSIONS = 384; -var CONCEPT_CACHE_VERSION = 1; +var CONCEPT_CACHE_VERSION = 2; var CONCEPT_CACHE_MAX_BYTES = 512 * 1024 * 1024; var CONCEPT_TIMEOUT_MS = 10 * 60000; var MAX_CONCEPT_TIMEOUT_MS = 60 * 60000; @@ -222,7 +222,7 @@ async function enforceConceptCacheLimit(root, maximumBytes = CONCEPT_CACHE_MAX_B } // src/concept-inference.ts -var INFERENCE_BATCH_SIZE = 16; +var INFERENCE_BATCH_SIZE = 4; function safeUtf16End(text, end) { if (end <= 0 || end >= text.length) return end; @@ -247,6 +247,8 @@ function binarySearchBudget(span) { return 2 * Math.max(span, 1) + 32; } function maximumTokenSafeEnd(extractor, prefix, text, start) { + if (tokenCount(extractor, `${prefix}${text.slice(start)}`) <= CONCEPT_MODEL_TOKENS) + return text.length; let low = start + 1; let high = text.length; let accepted = start; @@ -312,25 +314,49 @@ function tokenSafeWindows(extractor, pending) { async function embedConceptInputs(extractor, pending, callbacks = {}) { const layouts = pending.map((item) => tokenSafeWindows(extractor, item)); const inputs = layouts.flatMap((windows) => windows.map((window) => window.input)); - const vectors = []; + const vectors = Array(inputs.length); const layoutEnds = []; + const remainingWindows = layouts.map((windows) => windows.length); let layoutEnd = 0; for (const windows of layouts) { layoutEnd += windows.length; layoutEnds.push(layoutEnd); } + const orderedInputs = inputs.map((input, index) => ({ input, index, tokens: tokenCount(extractor, input) })).toSorted((left, right) => right.tokens - left.tokens || left.index - right.index); + const vectorFor = (index) => { + const vector = vectors[index]; + if (!vector) + throw new Error("Missing concept embedding vector"); + return vector; + }; + const ownerOf = (index) => { + let low = 0; + let high = layoutEnds.length; + while (low < high) { + const middle = Math.floor((low + high) / 2); + if (index < layoutEnds[middle]) + high = middle; + else + low = middle + 1; + } + return low; + }; let completedEmbedding = 0; - for (let offset = 0;offset < inputs.length; offset += INFERENCE_BATCH_SIZE) { - const batch = inputs.slice(offset, offset + INFERENCE_BATCH_SIZE); + let completedWindows = 0; + for (let offset = 0;offset < orderedInputs.length; ) { + const orderedBatch = orderedInputs.slice(offset, offset + INFERENCE_BATCH_SIZE); + const batch = orderedBatch.map((item) => item.input); const tensor = await extractor(batch, { pooling: "mean", normalize: true }); if (tensor.data.length !== batch.length * CONCEPT_EMBEDDING_DIMENSIONS) throw new Error("Unexpected concept embedding dimensions"); for (let index = 0;index < batch.length; index += 1) { const start = index * CONCEPT_EMBEDDING_DIMENSIONS; - vectors.push(Array.from(tensor.data.slice(start, start + CONCEPT_EMBEDDING_DIMENSIONS), Number)); + const inputIndex = orderedBatch[index].index; + vectors[inputIndex] = Array.from(tensor.data.slice(start, start + CONCEPT_EMBEDDING_DIMENSIONS), Number); + remainingWindows[ownerOf(inputIndex)]--; } - const completedWindows = Math.min(offset + batch.length, inputs.length); - while (layoutEnds[completedEmbedding] !== undefined && layoutEnds[completedEmbedding] <= completedWindows) { + completedWindows += batch.length; + while (remainingWindows[completedEmbedding] !== undefined && remainingWindows[completedEmbedding] === 0) { const start = completedEmbedding === 0 ? 0 : layoutEnds[completedEmbedding - 1]; const item = pending[completedEmbedding]; if (!item) @@ -340,12 +366,13 @@ async function embedConceptInputs(extractor, pending, callbacks = {}) { windows: (layouts[completedEmbedding] ?? []).map((window, index) => ({ start: window.start, end: window.end, - vector: vectors[start + index] ?? [] + vector: vectorFor(start + index) })) }, completedEmbedding + 1, pending.length); completedEmbedding += 1; } callbacks.onBatch?.(completedWindows, inputs.length); + offset += batch.length; } let vectorIndex = 0; return pending.map((item, index) => ({ @@ -353,7 +380,7 @@ async function embedConceptInputs(extractor, pending, callbacks = {}) { windows: (layouts[index] ?? []).map((window) => ({ start: window.start, end: window.end, - vector: vectors[vectorIndex++] ?? [] + vector: vectorFor(vectorIndex++) })) })); } @@ -504,7 +531,7 @@ async function search() { local_files_only: true, dtype: "q8", device: "cpu", - session_options: { intraOpNumThreads: 2, interOpNumThreads: 1 } + session_options: { intraOpNumThreads: 4, interOpNumThreads: 1 } }); try { created = await embedConceptInputs(extractor, missing, { diff --git a/plugins/sift-light/kimi.plugin.json b/plugins/sift-light/kimi.plugin.json index 3a1086b7..69b6f2c4 100644 --- a/plugins/sift-light/kimi.plugin.json +++ b/plugins/sift-light/kimi.plugin.json @@ -1,6 +1,6 @@ { "name": "sift-light", - "version": "1.0.0", + "version": "1.0.1", "description": "Require sift-light for conventional local searches while keeping development tools available.", "author": { "name": "baoer" @@ -13,7 +13,7 @@ "args": [ "--yes", "--package", - "sift-light-runtime@npm:sift-light@1.0.0", + "sift-light-runtime@npm:sift-light@1.0.1", "sift-light-mcp", "--stdio" ], diff --git a/plugins/sift-light/omp-extension.mjs b/plugins/sift-light/omp-extension.mjs index bf223803..713b35a5 100644 --- a/plugins/sift-light/omp-extension.mjs +++ b/plugins/sift-light/omp-extension.mjs @@ -1814,7 +1814,7 @@ function createCtagsStructureProvider(options = {}) { // package.json var package_default = { name: "sift-light", - version: "1.0.0", + version: "1.0.1", description: "Context-efficient local search for files, documents, notes and logs across Pi, OMP and MCP clients", keywords: [ "ai-agent", @@ -2185,7 +2185,7 @@ var CONCEPT_MODEL = "Xenova/multilingual-e5-small"; var CONCEPT_REVISION = "761b726dd34fb83930e26aab4e9ac3899aa1fa78"; var MAX_CONCEPT_CHARS = 1000; var CONCEPT_PASSAGE_OVERLAP_CHARS = 160; -var CONCEPT_CACHE_VERSION = 1; +var CONCEPT_CACHE_VERSION = 2; var CONCEPT_CACHE_MAX_BYTES = 512 * 1024 * 1024; var CONCEPT_TIMEOUT_MS = 10 * 60000; var MIN_CONCEPT_TIMEOUT_MS = 1000; @@ -4687,6 +4687,11 @@ function scoreProfile(scores) { spread: top - min }; } +function conceptRankingScore(cosine, passageLength) { + const referenceLength = 500; + const maximumCorrection = 0.02; + return cosine - maximumCorrection * (1 - Math.sqrt(Math.min(passageLength / referenceLength, 1))); +} function passage(document2, start2) { let end = Math.min(document2.text.length, start2 + MAX_CONCEPT_CHARS); if (end < document2.text.length) { @@ -4945,6 +4950,7 @@ async function runConceptSearch(input, access, infer, onProgress, options = {}) const similarity = inferred.scores[index]; if (similarity === undefined) throw new Error("Missing concept similarity"); + const rankingScore = conceptRankingScore(similarity, item.text.length); const evidence = rangeEvidence(item.document, item.range); return { path: item.document.path, @@ -4957,7 +4963,8 @@ async function runConceptSearch(input, access, infer, onProgress, options = {}) kind: "concept-candidate", certainty: "candidate", score: similarity, - rankingReason: "local multilingual E5 cosine similarity; relevance candidate, no binding or execution claim", + rankingScore, + rankingReason: "local multilingual E5 cosine similarity with bounded short-passage rank correction; relevance candidate, no binding or execution claim", model: CONCEPT_MODEL, revision: CONCEPT_REVISION, tokenTruncated: false, @@ -4966,7 +4973,7 @@ async function runConceptSearch(input, access, infer, onProgress, options = {}) } }; }); - const ranked = [...result.items, ...batchItems].toSorted((a, b) => Number(b.details?.score) - Number(a.details?.score) || a.path.localeCompare(b.path) || a.line - b.line); + const ranked = [...result.items, ...batchItems].toSorted((a, b) => Number(b.details?.rankingScore) - Number(a.details?.rankingScore) || a.path.localeCompare(b.path) || a.line - b.line); if (ranked.length > MAX_ANALYSIS_RESULTS) retentionTruncated = true; result.items = ranked.slice(0, MAX_ANALYSIS_RESULTS); @@ -9312,7 +9319,7 @@ function requestBody(query, candidates, model) { for (const candidate of candidates) { questions[candidate.id] = { type: "choice", - instructions: "Classify the candidate by what it actually does for the requested behavior. Judge the code excerpt, not just matching words.", + instructions: `Classify only candidate ${candidate.id} in state.candidates for state.query. Judge that candidate's excerpt by its actual behavior, not by matching words or the other candidates.`, criteria: { "implementation-candidate": "The excerpt appears to implement the requested behavior or its core decision/side effect.", "caller-candidate": "The excerpt invokes or wires an implementation but does not implement the behavior itself.", @@ -9781,6 +9788,9 @@ async function combineHybridSearch(scan, execution, access, conceptLimit, query, const deduplicationCoverage = scan.snapshotComplete && literal.sourceCoverage === "complete" && conceptSourceCoverage === "complete" ? "complete" : "partial"; const partial = !scan.snapshotComplete || concept.partial || conceptCoverage === "skipped" || conceptSourceCoverage === "partial" || literal.sourceCoverage === "partial" || deduplicationCoverage === "partial" || judgedConcept.semanticJudge?.status === "failed" || judgedConcept.semanticJudge?.status === "partial"; const selectionReason = conceptCandidatesOmitted ? `Hybrid concept limit retained the top ${String(selectedConcept.length)} of ${String(eligibleConcept.length)} non-overlapping semantic candidates` : undefined; + const firstScore = Number(eligibleConcept[0]?.details?.rankingScore ?? eligibleConcept[0]?.details?.score); + const secondScore = Number(eligibleConcept[1]?.details?.rankingScore ?? eligibleConcept[1]?.details?.score); + const closeRanking = Number.isFinite(firstScore) && Number.isFinite(secondScore) && firstScore - secondScore < 0.01; return { kind: "hybrid", unit: "evidence-items", @@ -9792,6 +9802,9 @@ async function combineHybridSearch(scan, execution, access, conceptLimit, query, ...literal.reasons, ...execution.sourceGeneration.reasons, ...selectionReason ? [selectionReason] : [], + ...closeRanking ? [ + "Semantic ranks are close; verify the leading candidates with source inspection or literal terms." + ] : [], ...judgedConcept.semanticJudge?.reason ? [judgedConcept.semanticJudge.reason] : [] ], filesRead: (concept.filesRead ?? 0) + access.filesRead, @@ -10215,15 +10228,23 @@ function schemaError(input, field, reason) { recovery: { action: "manual", reason } }, `${field} is invalid: ${reason}`); } +function plainFilePatternRecovery(mode, input, invalid) { + return mode === "files" && invalid.includes("pattern") && input.query === undefined && typeof input.pattern === "string" && input.pattern.trim().length > 0 && input.pattern.length <= 256 && input.pattern.isWellFormed() && !/[\\^$.*+?()[\]{}|\r\n\0]/u.test(input.pattern); +} function safeNextRequest(input, mode, invalid) { if (input.redact === true && containsSensitiveText(input)) return; const safe = new Set(SAFE_DROP_FIELDS[mode] ?? []); + const plainFilePattern = plainFilePatternRecovery(mode, input, invalid); + if (plainFilePattern) + safe.add("pattern"); if (invalid.some((field) => !safe.has(field))) return; const next = { ...input }; for (const field of invalid) delete next[field]; + if (plainFilePattern) + next.query = input.pattern; if (selectorIssuesFor(next, mode).length > 0) return; try { @@ -10252,20 +10273,26 @@ function fieldsError(input, mode, invalid, selectorIssues = []) { }); const visibleFields = visibleInvalid.map((field) => boundedField(field)).join(", "); const reason = omitted > 0 ? `mode=${mode} does not accept ${visibleFields} and ${String(omitted)} additional field(s)` : `mode=${mode} does not accept ${visibleFields}`; - const flatRule = `Fields are flat: pass them at the top level, not inside a per-mode object. mode=${mode} accepts: ${modeFields(mode).join(", ")}.`; + const nestedModeObject = invalid.includes(mode); + const flatRule = nestedModeObject ? `Fields are flat: pass them at the top level, not inside a per-mode object. mode=${mode} accepts: ${modeFields(mode).join(", ")}.` : ""; + const filesPatternRule = mode === "files" && invalid.includes("pattern") ? "For filename or path discovery, put the literal name text in query; pattern is a content-search regex, so check any regex syntax before copying it." : ""; return new RequestContractError({ code: "E_MODE_FIELDS", mode, issues, recovery: nextRequest ? { action: "retry", - reason: `Remove only ${visibleFields} and copy the exact nextRequest; all other fields are preserved.`, + reason: plainFilePatternRecovery(mode, input, invalid) ? "For filename discovery, move the plain text from pattern to query and copy nextRequest; all filters are preserved." : `Remove only ${visibleFields} and copy the exact nextRequest; all other fields are preserved.`, nextRequest } : { action: "manual", - reason: `${reason}; choose the mode explicitly or remove the fields yourself without changing the requested scope. ${flatRule}` + reason: [ + reason, + filesPatternRule || "Choose the mode explicitly or remove unsupported fields without changing the requested scope.", + flatRule + ].filter(Boolean).join(" ") } - }, nextRequest ? `${reason}; retry the exact nextRequest without repeating the original query.` : `${reason}; no semantics-preserving automatic request is available. ${flatRule} Preserve valid path, filters, redact and cursor fields when choosing the next request.`); + }, nextRequest ? `${reason}; ${plainFilePatternRecovery(mode, input, invalid) ? "move plain filename text to query" : "retry the exact nextRequest"}.` : `${reason}; no semantics-preserving automatic request is available. ${filesPatternRule || flatRule || "Check the mode's accepted fields."}`); } function selectorIssuesFor(input, mode) { if ((mode === "auto" || mode === "summary" || mode === "matches") && typeof input.cursor === "string" && input.cursor.trim().length > 0 && !input.cursor.includes(".analysis")) { @@ -20750,25 +20777,23 @@ function modelErrorText(error) { function requestContractErrorText(error) { return requestContractProjection(error).text; } -function projectRequestContract(projected, serialized, message) { - const prefix = `sift-light failed: request rejected [${projected.code}]: ${message}`; +function projectRequestContract(projected, serialized) { const recovery = projected.recovery; const next = recovery.nextRequest ? ` Copy the nested recovery.nextRequest object unchanged; do not repeat the original query.` : ` -Recovery action: ${recovery.action}. ${recovery.reason}`; +Recovery action: ${recovery.action}.`; return ` -Error details: ${serialized} -${prefix}${next}`; +sift-light failed: request rejected [${projected.code}]. +Error details: ${serialized}${next}`; } function requestContractProjection(error) { const details = boundedRequestContractDetails(error.details); const serializedDetails = JSON.stringify(details); - const boundedMessage = error.message.toWellFormed().slice(0, 1024); - const result = projectRequestContract(details, serializedDetails, boundedMessage); + const result = projectRequestContract(details, serializedDetails); if (Buffer.byteLength(result) <= MAX_REQUEST_RECOVERY_BYTES) return { details, text: result }; const compact = { - code: boundedMessage.length > 0 ? details.code : "E_REQUEST_CONTRACT_PAYLOAD", + code: details.code, ...details.mode ? { mode: details.mode } : {}, issues: [ { @@ -20781,7 +20806,7 @@ function requestContractProjection(error) { reason: "Exact recovery was omitted; correct the request explicitly." } }; - const compactText = projectRequestContract(compact, JSON.stringify(compact), "The request-contract error exceeded the bounded payload budget; correct the request explicitly."); + const compactText = projectRequestContract(compact, JSON.stringify(compact)); return { details: compact, text: compactText }; } diff --git a/plugins/sift-light/package.json b/plugins/sift-light/package.json index 009dc955..4ac4f7c0 100644 --- a/plugins/sift-light/package.json +++ b/plugins/sift-light/package.json @@ -1,6 +1,6 @@ { "name": "sift-light", - "version": "1.0.0", + "version": "1.0.1", "private": true, "type": "module", "omp": { diff --git a/src/concept-inference.ts b/src/concept-inference.ts index 9e6f7943..6b874682 100644 --- a/src/concept-inference.ts +++ b/src/concept-inference.ts @@ -21,7 +21,9 @@ export interface ConceptEmbeddingCallbacks { ) => void | Promise; } -const INFERENCE_BATCH_SIZE = 16; +// Bound each ONNX call to four model windows. Larger native batches raised peak +// RSS without a useful cold-start gain on the measured workload. +const INFERENCE_BATCH_SIZE = 4; /** Snap a candidate UTF-16 index so it never splits a surrogate pair. */ export function safeUtf16End(text: string, end: number): number { @@ -62,6 +64,10 @@ export function maximumTokenSafeEnd( text: string, start: number, ): number { + // Most source passages already fit the model window. Avoid repeatedly + // tokenizing prefixes of the same passage in the binary-search path. + if (tokenCount(extractor, `${prefix}${text.slice(start)}`) <= CONCEPT_MODEL_TOKENS) + return text.length; let low = start + 1; let high = text.length; let accepted = start; @@ -142,30 +148,56 @@ export async function embedConceptInputs( ): Promise { const layouts = pending.map((item) => tokenSafeWindows(extractor, item)); const inputs = layouts.flatMap((windows) => windows.map((window) => window.input)); - const vectors: number[][] = []; + const vectors: (number[] | undefined)[] = Array(inputs.length); const layoutEnds: number[] = []; + const remainingWindows = layouts.map((windows) => windows.length); let layoutEnd = 0; for (const windows of layouts) { layoutEnd += windows.length; layoutEnds.push(layoutEnd); } + const orderedInputs = inputs + .map((input, index) => ({ input, index, tokens: tokenCount(extractor, input) })) + // Start with the largest shapes so the native arena can reuse allocations + // for smaller batches instead of retaining a growing sequence of arenas. + .toSorted((left, right) => right.tokens - left.tokens || left.index - right.index); + const vectorFor = (index: number): number[] => { + const vector = vectors[index]; + if (!vector) throw new Error("Missing concept embedding vector"); + return vector; + }; + const ownerOf = (index: number): number => { + let low = 0; + let high = layoutEnds.length; + while (low < high) { + const middle = Math.floor((low + high) / 2); + if (index < layoutEnds[middle]!) high = middle; + else low = middle + 1; + } + return low; + }; let completedEmbedding = 0; - for (let offset = 0; offset < inputs.length; offset += INFERENCE_BATCH_SIZE) { - const batch = inputs.slice(offset, offset + INFERENCE_BATCH_SIZE); + let completedWindows = 0; + for (let offset = 0; offset < orderedInputs.length;) { + const orderedBatch = orderedInputs.slice(offset, offset + INFERENCE_BATCH_SIZE); + const batch = orderedBatch.map((item) => item.input); // oxlint-disable-next-line no-await-in-loop -- bounded batches cap native tensor memory. const tensor = await extractor(batch, { pooling: "mean", normalize: true }); if (tensor.data.length !== batch.length * CONCEPT_EMBEDDING_DIMENSIONS) throw new Error("Unexpected concept embedding dimensions"); for (let index = 0; index < batch.length; index += 1) { const start = index * CONCEPT_EMBEDDING_DIMENSIONS; - vectors.push( - Array.from(tensor.data.slice(start, start + CONCEPT_EMBEDDING_DIMENSIONS), Number), + const inputIndex = orderedBatch[index]!.index; + vectors[inputIndex] = Array.from( + tensor.data.slice(start, start + CONCEPT_EMBEDDING_DIMENSIONS), + Number, ); + remainingWindows[ownerOf(inputIndex)]!--; } - const completedWindows = Math.min(offset + batch.length, inputs.length); + completedWindows += batch.length; while ( - layoutEnds[completedEmbedding] !== undefined && - layoutEnds[completedEmbedding]! <= completedWindows + remainingWindows[completedEmbedding] !== undefined && + remainingWindows[completedEmbedding] === 0 ) { const start = completedEmbedding === 0 ? 0 : layoutEnds[completedEmbedding - 1]!; const item = pending[completedEmbedding]; @@ -177,7 +209,7 @@ export async function embedConceptInputs( windows: (layouts[completedEmbedding] ?? []).map((window, index) => ({ start: window.start, end: window.end, - vector: vectors[start + index] ?? [], + vector: vectorFor(start + index), })), }, completedEmbedding + 1, @@ -186,6 +218,7 @@ export async function embedConceptInputs( completedEmbedding += 1; } callbacks.onBatch?.(completedWindows, inputs.length); + offset += batch.length; } let vectorIndex = 0; return pending.map((item, index) => ({ @@ -193,7 +226,7 @@ export async function embedConceptInputs( windows: (layouts[index] ?? []).map((window) => ({ start: window.start, end: window.end, - vector: vectors[vectorIndex++] ?? [], + vector: vectorFor(vectorIndex++), })), })); } diff --git a/src/concept-model.ts b/src/concept-model.ts index bb0a86e6..bd94ba1b 100644 --- a/src/concept-model.ts +++ b/src/concept-model.ts @@ -8,10 +8,10 @@ export const CONCEPT_MODEL = "Xenova/multilingual-e5-small"; export const CONCEPT_REVISION = "761b726dd34fb83930e26aab4e9ac3899aa1fa78"; export const MAX_CONCEPT_CHARS = 1_000; export const CONCEPT_PASSAGE_OVERLAP_CHARS = 160; -export const CONCEPT_MODEL_TOKENS = 512; +export const CONCEPT_MODEL_TOKENS = 256; export const CONCEPT_WINDOW_OVERLAP_TOKENS = 64; export const CONCEPT_EMBEDDING_DIMENSIONS = 384; -export const CONCEPT_CACHE_VERSION = 1; +export const CONCEPT_CACHE_VERSION = 2; export const CONCEPT_CACHE_MAX_BYTES = 512 * 1024 * 1024; export const CONCEPT_TIMEOUT_MS = 10 * 60_000; export const MIN_CONCEPT_TIMEOUT_MS = 1_000; diff --git a/src/concept-search.ts b/src/concept-search.ts index d8113d03..fd6a4270 100644 --- a/src/concept-search.ts +++ b/src/concept-search.ts @@ -86,6 +86,14 @@ function scoreProfile(scores: readonly number[]): ConceptScoreProfile { }; } +/** E5 cosine scores favor very short, generic passages in close races. Keep the + * raw cosine as evidence and apply a bounded length correction only to rank. */ +export function conceptRankingScore(cosine: number, passageLength: number): number { + const referenceLength = 500; + const maximumCorrection = 0.02; + return cosine - maximumCorrection * (1 - Math.sqrt(Math.min(passageLength / referenceLength, 1))); +} + function passage(document: SourceDocument, start: number): { value: Passage; next: number } { let end = Math.min(document.text.length, start + MAX_CONCEPT_CHARS); if (end < document.text.length) { @@ -433,6 +441,7 @@ async function runConceptSearch( const batchItems = passages.map((item, index) => { const similarity = inferred.scores[index]; if (similarity === undefined) throw new Error("Missing concept similarity"); + const rankingScore = conceptRankingScore(similarity, item.text.length); const evidence = rangeEvidence(item.document, item.range); return { path: item.document.path, @@ -445,8 +454,9 @@ async function runConceptSearch( kind: "concept-candidate", certainty: "candidate", score: similarity, + rankingScore, rankingReason: - "local multilingual E5 cosine similarity; relevance candidate, no binding or execution claim", + "local multilingual E5 cosine similarity with bounded short-passage rank correction; relevance candidate, no binding or execution claim", model: CONCEPT_MODEL, revision: CONCEPT_REVISION, tokenTruncated: false, @@ -457,7 +467,7 @@ async function runConceptSearch( }); const ranked = [...result.items, ...batchItems].toSorted( (a, b) => - Number(b.details?.score) - Number(a.details?.score) || + Number(b.details?.rankingScore) - Number(a.details?.rankingScore) || a.path.localeCompare(b.path) || a.line - b.line, ); diff --git a/src/concept-worker.mjs b/src/concept-worker.mjs index d09872aa..84eabc6a 100755 --- a/src/concept-worker.mjs +++ b/src/concept-worker.mjs @@ -27,10 +27,10 @@ class SiftLightError extends Error { var CONCEPT_MODEL = "Xenova/multilingual-e5-small"; var CONCEPT_REVISION = "761b726dd34fb83930e26aab4e9ac3899aa1fa78"; var MAX_CONCEPT_CHARS = 1000; -var CONCEPT_MODEL_TOKENS = 512; +var CONCEPT_MODEL_TOKENS = 256; var CONCEPT_WINDOW_OVERLAP_TOKENS = 64; var CONCEPT_EMBEDDING_DIMENSIONS = 384; -var CONCEPT_CACHE_VERSION = 1; +var CONCEPT_CACHE_VERSION = 2; var CONCEPT_CACHE_MAX_BYTES = 512 * 1024 * 1024; var CONCEPT_TIMEOUT_MS = 10 * 60000; var MAX_CONCEPT_TIMEOUT_MS = 60 * 60000; @@ -222,7 +222,7 @@ async function enforceConceptCacheLimit(root, maximumBytes = CONCEPT_CACHE_MAX_B } // src/concept-inference.ts -var INFERENCE_BATCH_SIZE = 16; +var INFERENCE_BATCH_SIZE = 4; function safeUtf16End(text, end) { if (end <= 0 || end >= text.length) return end; @@ -247,6 +247,8 @@ function binarySearchBudget(span) { return 2 * Math.max(span, 1) + 32; } function maximumTokenSafeEnd(extractor, prefix, text, start) { + if (tokenCount(extractor, `${prefix}${text.slice(start)}`) <= CONCEPT_MODEL_TOKENS) + return text.length; let low = start + 1; let high = text.length; let accepted = start; @@ -312,25 +314,49 @@ function tokenSafeWindows(extractor, pending) { async function embedConceptInputs(extractor, pending, callbacks = {}) { const layouts = pending.map((item) => tokenSafeWindows(extractor, item)); const inputs = layouts.flatMap((windows) => windows.map((window) => window.input)); - const vectors = []; + const vectors = Array(inputs.length); const layoutEnds = []; + const remainingWindows = layouts.map((windows) => windows.length); let layoutEnd = 0; for (const windows of layouts) { layoutEnd += windows.length; layoutEnds.push(layoutEnd); } + const orderedInputs = inputs.map((input, index) => ({ input, index, tokens: tokenCount(extractor, input) })).toSorted((left, right) => right.tokens - left.tokens || left.index - right.index); + const vectorFor = (index) => { + const vector = vectors[index]; + if (!vector) + throw new Error("Missing concept embedding vector"); + return vector; + }; + const ownerOf = (index) => { + let low = 0; + let high = layoutEnds.length; + while (low < high) { + const middle = Math.floor((low + high) / 2); + if (index < layoutEnds[middle]) + high = middle; + else + low = middle + 1; + } + return low; + }; let completedEmbedding = 0; - for (let offset = 0;offset < inputs.length; offset += INFERENCE_BATCH_SIZE) { - const batch = inputs.slice(offset, offset + INFERENCE_BATCH_SIZE); + let completedWindows = 0; + for (let offset = 0;offset < orderedInputs.length; ) { + const orderedBatch = orderedInputs.slice(offset, offset + INFERENCE_BATCH_SIZE); + const batch = orderedBatch.map((item) => item.input); const tensor = await extractor(batch, { pooling: "mean", normalize: true }); if (tensor.data.length !== batch.length * CONCEPT_EMBEDDING_DIMENSIONS) throw new Error("Unexpected concept embedding dimensions"); for (let index = 0;index < batch.length; index += 1) { const start = index * CONCEPT_EMBEDDING_DIMENSIONS; - vectors.push(Array.from(tensor.data.slice(start, start + CONCEPT_EMBEDDING_DIMENSIONS), Number)); + const inputIndex = orderedBatch[index].index; + vectors[inputIndex] = Array.from(tensor.data.slice(start, start + CONCEPT_EMBEDDING_DIMENSIONS), Number); + remainingWindows[ownerOf(inputIndex)]--; } - const completedWindows = Math.min(offset + batch.length, inputs.length); - while (layoutEnds[completedEmbedding] !== undefined && layoutEnds[completedEmbedding] <= completedWindows) { + completedWindows += batch.length; + while (remainingWindows[completedEmbedding] !== undefined && remainingWindows[completedEmbedding] === 0) { const start = completedEmbedding === 0 ? 0 : layoutEnds[completedEmbedding - 1]; const item = pending[completedEmbedding]; if (!item) @@ -340,12 +366,13 @@ async function embedConceptInputs(extractor, pending, callbacks = {}) { windows: (layouts[completedEmbedding] ?? []).map((window, index) => ({ start: window.start, end: window.end, - vector: vectors[start + index] ?? [] + vector: vectorFor(start + index) })) }, completedEmbedding + 1, pending.length); completedEmbedding += 1; } callbacks.onBatch?.(completedWindows, inputs.length); + offset += batch.length; } let vectorIndex = 0; return pending.map((item, index) => ({ @@ -353,7 +380,7 @@ async function embedConceptInputs(extractor, pending, callbacks = {}) { windows: (layouts[index] ?? []).map((window) => ({ start: window.start, end: window.end, - vector: vectors[vectorIndex++] ?? [] + vector: vectorFor(vectorIndex++) })) })); } @@ -504,7 +531,7 @@ async function search() { local_files_only: true, dtype: "q8", device: "cpu", - session_options: { intraOpNumThreads: 2, interOpNumThreads: 1 } + session_options: { intraOpNumThreads: 4, interOpNumThreads: 1 } }); try { created = await embedConceptInputs(extractor, missing, { diff --git a/src/concept-worker.ts b/src/concept-worker.ts index 0b239064..285979d7 100644 --- a/src/concept-worker.ts +++ b/src/concept-worker.ts @@ -140,7 +140,7 @@ async function search(): Promise { local_files_only: true, dtype: "q8", device: "cpu", - session_options: { intraOpNumThreads: 2, interOpNumThreads: 1 }, + session_options: { intraOpNumThreads: 4, interOpNumThreads: 1 }, }); try { created = await embedConceptInputs(extractor, missing, { diff --git a/src/hybrid-search.ts b/src/hybrid-search.ts index 5086a5cc..851e22be 100644 --- a/src/hybrid-search.ts +++ b/src/hybrid-search.ts @@ -274,6 +274,14 @@ export async function combineHybridSearch( const selectionReason = conceptCandidatesOmitted ? `Hybrid concept limit retained the top ${String(selectedConcept.length)} of ${String(eligibleConcept.length)} non-overlapping semantic candidates` : undefined; + const firstScore = Number( + eligibleConcept[0]?.details?.rankingScore ?? eligibleConcept[0]?.details?.score, + ); + const secondScore = Number( + eligibleConcept[1]?.details?.rankingScore ?? eligibleConcept[1]?.details?.score, + ); + const closeRanking = + Number.isFinite(firstScore) && Number.isFinite(secondScore) && firstScore - secondScore < 0.01; return { kind: "hybrid", unit: "evidence-items", @@ -285,6 +293,11 @@ export async function combineHybridSearch( ...literal.reasons, ...execution.sourceGeneration.reasons, ...(selectionReason ? [selectionReason] : []), + ...(closeRanking + ? [ + "Semantic ranks are close; verify the leading candidates with source inspection or literal terms.", + ] + : []), ...(judgedConcept.semanticJudge?.reason ? [judgedConcept.semanticJudge.reason] : []), ], filesRead: (concept.filesRead ?? 0) + access.filesRead, diff --git a/src/mcp-model-output.ts b/src/mcp-model-output.ts index 27a8cc11..9f8966cd 100644 --- a/src/mcp-model-output.ts +++ b/src/mcp-model-output.ts @@ -7,7 +7,48 @@ function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } +function compactSemanticMetadata(details: SiftLightDetails, analysis: AnalysisDetails): string[] { + const counts = analysis.counts; + const stats = analysis.stats; + const judge = analysis.semanticJudge; + const coverage = analysis.coverage; + return [ + ...(counts || stats + ? [ + `Search: ${String(counts?.filesAdmitted ?? stats?.filesAdmitted ?? 0)} files, ${String(stats?.passagesRanked ?? counts?.passagesQueued ?? 0)} passages; ${String(stats?.elapsedMs ?? 0)} ms; peak inference RSS ${String(stats?.inferencePeakRssBytes ?? 0)} bytes; cache ${String(stats?.conceptCacheHits ?? 0)} hits/${String(stats?.conceptCacheMisses ?? 0)} misses.`, + ] + : []), + ...(coverage + ? [ + `Coverage: ${Object.entries(coverage) + .map(([name, status]) => `${name}=${status}`) + .join(", ")}.`, + ] + : []), + ...(analysis.scope + ? [`Scope: ${analysis.scope.path}; ignore=${analysis.scope.ignorePolicy}.`] + : []), + ...(analysis.sourceGeneration + ? [ + `Source: ${analysis.sourceGeneration.verification}; ${String(analysis.sourceGeneration.filesUnavailable)} unavailable.`, + ] + : []), + ...(judge + ? [ + `Semantic judge: ${judge.status}; ${String(judge.judgedCandidates)}/${String(judge.candidatesConsidered)} judged, ${String(judge.candidatesUnjudged)} unjudged; batches ${String(judge.batchesCompleted)}/${String(judge.batchesAttempted)} completed; classes ${JSON.stringify(judge.classificationCounts)}${judge.reason ? `; ${judge.reason}` : ""}.`, + ] + : []), + ...(details.operation + ? [`Operation: ${details.operation.state}; id=${details.operation.id}.`] + : []), + ...analysis.reasons.map((reason) => `[${reason}]`), + ...(details.redactionApplied ? ["[Display redaction applied.]"] : []), + ]; +} + function compactMetadata(details: SiftLightDetails, analysis: AnalysisDetails): string[] { + if (analysis.kind === "concept" || analysis.kind === "hybrid") + return compactSemanticMetadata(details, analysis); return [ ...(analysis.statistics ? formatStatistics(analysis.statistics) : []), analysis.counts ? `Counts: ${JSON.stringify(analysis.counts)}` : undefined, diff --git a/src/mcp-server.mjs b/src/mcp-server.mjs index ad16ed23..6d56aa56 100755 --- a/src/mcp-server.mjs +++ b/src/mcp-server.mjs @@ -7,7 +7,7 @@ import { URL as URL2 } from "node:url"; // package.json var package_default = { name: "sift-light", - version: "1.0.0", + version: "1.0.1", description: "Context-efficient local search for files, documents, notes and logs across Pi, OMP and MCP clients", keywords: [ "ai-agent", @@ -292,7 +292,33 @@ function analysisExtraGroups(counts, termCounts, items) { function isRecord(value) { return typeof value === "object" && value !== null && !Array.isArray(value); } +function compactSemanticMetadata(details, analysis) { + const counts = analysis.counts; + const stats = analysis.stats; + const judge = analysis.semanticJudge; + const coverage = analysis.coverage; + return [ + ...counts || stats ? [ + `Search: ${String(counts?.filesAdmitted ?? stats?.filesAdmitted ?? 0)} files, ${String(stats?.passagesRanked ?? counts?.passagesQueued ?? 0)} passages; ${String(stats?.elapsedMs ?? 0)} ms; peak inference RSS ${String(stats?.inferencePeakRssBytes ?? 0)} bytes; cache ${String(stats?.conceptCacheHits ?? 0)} hits/${String(stats?.conceptCacheMisses ?? 0)} misses.` + ] : [], + ...coverage ? [ + `Coverage: ${Object.entries(coverage).map(([name, status]) => `${name}=${status}`).join(", ")}.` + ] : [], + ...analysis.scope ? [`Scope: ${analysis.scope.path}; ignore=${analysis.scope.ignorePolicy}.`] : [], + ...analysis.sourceGeneration ? [ + `Source: ${analysis.sourceGeneration.verification}; ${String(analysis.sourceGeneration.filesUnavailable)} unavailable.` + ] : [], + ...judge ? [ + `Semantic judge: ${judge.status}; ${String(judge.judgedCandidates)}/${String(judge.candidatesConsidered)} judged, ${String(judge.candidatesUnjudged)} unjudged; batches ${String(judge.batchesCompleted)}/${String(judge.batchesAttempted)} completed; classes ${JSON.stringify(judge.classificationCounts)}${judge.reason ? `; ${judge.reason}` : ""}.` + ] : [], + ...details.operation ? [`Operation: ${details.operation.state}; id=${details.operation.id}.`] : [], + ...analysis.reasons.map((reason) => `[${reason}]`), + ...details.redactionApplied ? ["[Display redaction applied.]"] : [] + ]; +} function compactMetadata(details, analysis) { + if (analysis.kind === "concept" || analysis.kind === "hybrid") + return compactSemanticMetadata(details, analysis); return [ ...analysis.statistics ? formatStatistics(analysis.statistics) : [], analysis.counts ? `Counts: ${JSON.stringify(analysis.counts)}` : undefined, @@ -996,15 +1022,23 @@ function schemaError(input, field, reason) { recovery: { action: "manual", reason } }, `${field} is invalid: ${reason}`); } +function plainFilePatternRecovery(mode, input, invalid) { + return mode === "files" && invalid.includes("pattern") && input.query === undefined && typeof input.pattern === "string" && input.pattern.trim().length > 0 && input.pattern.length <= 256 && input.pattern.isWellFormed() && !/[\\^$.*+?()[\]{}|\r\n\0]/u.test(input.pattern); +} function safeNextRequest(input, mode, invalid) { if (input.redact === true && containsSensitiveText(input)) return; const safe = new Set(SAFE_DROP_FIELDS[mode] ?? []); + const plainFilePattern = plainFilePatternRecovery(mode, input, invalid); + if (plainFilePattern) + safe.add("pattern"); if (invalid.some((field) => !safe.has(field))) return; const next = { ...input }; for (const field of invalid) delete next[field]; + if (plainFilePattern) + next.query = input.pattern; if (selectorIssuesFor(next, mode).length > 0) return; try { @@ -1033,20 +1067,26 @@ function fieldsError(input, mode, invalid, selectorIssues = []) { }); const visibleFields = visibleInvalid.map((field) => boundedField(field)).join(", "); const reason = omitted > 0 ? `mode=${mode} does not accept ${visibleFields} and ${String(omitted)} additional field(s)` : `mode=${mode} does not accept ${visibleFields}`; - const flatRule = `Fields are flat: pass them at the top level, not inside a per-mode object. mode=${mode} accepts: ${modeFields(mode).join(", ")}.`; + const nestedModeObject = invalid.includes(mode); + const flatRule = nestedModeObject ? `Fields are flat: pass them at the top level, not inside a per-mode object. mode=${mode} accepts: ${modeFields(mode).join(", ")}.` : ""; + const filesPatternRule = mode === "files" && invalid.includes("pattern") ? "For filename or path discovery, put the literal name text in query; pattern is a content-search regex, so check any regex syntax before copying it." : ""; return new RequestContractError({ code: "E_MODE_FIELDS", mode, issues, recovery: nextRequest ? { action: "retry", - reason: `Remove only ${visibleFields} and copy the exact nextRequest; all other fields are preserved.`, + reason: plainFilePatternRecovery(mode, input, invalid) ? "For filename discovery, move the plain text from pattern to query and copy nextRequest; all filters are preserved." : `Remove only ${visibleFields} and copy the exact nextRequest; all other fields are preserved.`, nextRequest } : { action: "manual", - reason: `${reason}; choose the mode explicitly or remove the fields yourself without changing the requested scope. ${flatRule}` + reason: [ + reason, + filesPatternRule || "Choose the mode explicitly or remove unsupported fields without changing the requested scope.", + flatRule + ].filter(Boolean).join(" ") } - }, nextRequest ? `${reason}; retry the exact nextRequest without repeating the original query.` : `${reason}; no semantics-preserving automatic request is available. ${flatRule} Preserve valid path, filters, redact and cursor fields when choosing the next request.`); + }, nextRequest ? `${reason}; ${plainFilePatternRecovery(mode, input, invalid) ? "move plain filename text to query" : "retry the exact nextRequest"}.` : `${reason}; no semantics-preserving automatic request is available. ${filesPatternRule || flatRule || "Check the mode's accepted fields."}`); } function selectorIssuesFor(input, mode) { if ((mode === "auto" || mode === "summary" || mode === "matches") && typeof input.cursor === "string" && input.cursor.trim().length > 0 && !input.cursor.includes(".analysis")) { @@ -1282,25 +1322,23 @@ function modelErrorText(error) { function requestContractErrorText(error) { return requestContractProjection(error).text; } -function projectRequestContract(projected, serialized, message) { - const prefix = `sift-light failed: request rejected [${projected.code}]: ${message}`; +function projectRequestContract(projected, serialized) { const recovery = projected.recovery; const next = recovery.nextRequest ? ` Copy the nested recovery.nextRequest object unchanged; do not repeat the original query.` : ` -Recovery action: ${recovery.action}. ${recovery.reason}`; +Recovery action: ${recovery.action}.`; return ` -Error details: ${serialized} -${prefix}${next}`; +sift-light failed: request rejected [${projected.code}]. +Error details: ${serialized}${next}`; } function requestContractProjection(error) { const details = boundedRequestContractDetails(error.details); const serializedDetails = JSON.stringify(details); - const boundedMessage = error.message.toWellFormed().slice(0, 1024); - const result = projectRequestContract(details, serializedDetails, boundedMessage); + const result = projectRequestContract(details, serializedDetails); if (Buffer.byteLength(result) <= MAX_REQUEST_RECOVERY_BYTES) return { details, text: result }; const compact = { - code: boundedMessage.length > 0 ? details.code : "E_REQUEST_CONTRACT_PAYLOAD", + code: details.code, ...details.mode ? { mode: details.mode } : {}, issues: [ { @@ -1313,7 +1351,7 @@ function requestContractProjection(error) { reason: "Exact recovery was omitted; correct the request explicitly." } }; - const compactText = projectRequestContract(compact, JSON.stringify(compact), "The request-contract error exceeded the bounded payload budget; correct the request explicitly."); + const compactText = projectRequestContract(compact, JSON.stringify(compact)); return { details: compact, text: compactText }; } @@ -2819,7 +2857,7 @@ var CONCEPT_MODEL = "Xenova/multilingual-e5-small"; var CONCEPT_REVISION = "761b726dd34fb83930e26aab4e9ac3899aa1fa78"; var MAX_CONCEPT_CHARS = 1000; var CONCEPT_PASSAGE_OVERLAP_CHARS = 160; -var CONCEPT_CACHE_VERSION = 1; +var CONCEPT_CACHE_VERSION = 2; var CONCEPT_CACHE_MAX_BYTES = 512 * 1024 * 1024; var CONCEPT_TIMEOUT_MS = 10 * 60000; var MIN_CONCEPT_TIMEOUT_MS = 1000; @@ -5142,6 +5180,11 @@ function scoreProfile(scores) { spread: top - min }; } +function conceptRankingScore(cosine, passageLength) { + const referenceLength = 500; + const maximumCorrection = 0.02; + return cosine - maximumCorrection * (1 - Math.sqrt(Math.min(passageLength / referenceLength, 1))); +} function passage(document, start) { let end = Math.min(document.text.length, start + MAX_CONCEPT_CHARS); if (end < document.text.length) { @@ -5400,6 +5443,7 @@ async function runConceptSearch(input, access, infer, onProgress, options = {}) const similarity = inferred.scores[index]; if (similarity === undefined) throw new Error("Missing concept similarity"); + const rankingScore = conceptRankingScore(similarity, item.text.length); const evidence = rangeEvidence(item.document, item.range); return { path: item.document.path, @@ -5412,7 +5456,8 @@ async function runConceptSearch(input, access, infer, onProgress, options = {}) kind: "concept-candidate", certainty: "candidate", score: similarity, - rankingReason: "local multilingual E5 cosine similarity; relevance candidate, no binding or execution claim", + rankingScore, + rankingReason: "local multilingual E5 cosine similarity with bounded short-passage rank correction; relevance candidate, no binding or execution claim", model: CONCEPT_MODEL, revision: CONCEPT_REVISION, tokenTruncated: false, @@ -5421,7 +5466,7 @@ async function runConceptSearch(input, access, infer, onProgress, options = {}) } }; }); - const ranked = [...result.items, ...batchItems].toSorted((a, b) => Number(b.details?.score) - Number(a.details?.score) || a.path.localeCompare(b.path) || a.line - b.line); + const ranked = [...result.items, ...batchItems].toSorted((a, b) => Number(b.details?.rankingScore) - Number(a.details?.rankingScore) || a.path.localeCompare(b.path) || a.line - b.line); if (ranked.length > MAX_ANALYSIS_RESULTS) retentionTruncated = true; result.items = ranked.slice(0, MAX_ANALYSIS_RESULTS); @@ -9664,7 +9709,7 @@ function requestBody(query, candidates, model) { for (const candidate of candidates) { questions[candidate.id] = { type: "choice", - instructions: "Classify the candidate by what it actually does for the requested behavior. Judge the code excerpt, not just matching words.", + instructions: `Classify only candidate ${candidate.id} in state.candidates for state.query. Judge that candidate's excerpt by its actual behavior, not by matching words or the other candidates.`, criteria: { "implementation-candidate": "The excerpt appears to implement the requested behavior or its core decision/side effect.", "caller-candidate": "The excerpt invokes or wires an implementation but does not implement the behavior itself.", @@ -10133,6 +10178,9 @@ async function combineHybridSearch(scan, execution, access, conceptLimit, query, const deduplicationCoverage = scan.snapshotComplete && literal.sourceCoverage === "complete" && conceptSourceCoverage === "complete" ? "complete" : "partial"; const partial = !scan.snapshotComplete || concept.partial || conceptCoverage === "skipped" || conceptSourceCoverage === "partial" || literal.sourceCoverage === "partial" || deduplicationCoverage === "partial" || judgedConcept.semanticJudge?.status === "failed" || judgedConcept.semanticJudge?.status === "partial"; const selectionReason = conceptCandidatesOmitted ? `Hybrid concept limit retained the top ${String(selectedConcept.length)} of ${String(eligibleConcept.length)} non-overlapping semantic candidates` : undefined; + const firstScore = Number(eligibleConcept[0]?.details?.rankingScore ?? eligibleConcept[0]?.details?.score); + const secondScore = Number(eligibleConcept[1]?.details?.rankingScore ?? eligibleConcept[1]?.details?.score); + const closeRanking = Number.isFinite(firstScore) && Number.isFinite(secondScore) && firstScore - secondScore < 0.01; return { kind: "hybrid", unit: "evidence-items", @@ -10144,6 +10192,9 @@ async function combineHybridSearch(scan, execution, access, conceptLimit, query, ...literal.reasons, ...execution.sourceGeneration.reasons, ...selectionReason ? [selectionReason] : [], + ...closeRanking ? [ + "Semantic ranks are close; verify the leading candidates with source inspection or literal terms." + ] : [], ...judgedConcept.semanticJudge?.reason ? [judgedConcept.semanticJudge.reason] : [] ], filesRead: (concept.filesRead ?? 0) + access.filesRead, diff --git a/src/model-error.ts b/src/model-error.ts index 5bbed1e9..9f3241d5 100644 --- a/src/model-error.ts +++ b/src/model-error.ts @@ -73,19 +73,14 @@ export interface RequestContractProjection { text: string; } -function projectRequestContract( - projected: RequestContractDetails, - serialized: string, - message: string, -): string { - const prefix = `sift-light failed: request rejected [${projected.code}]: ${message}`; +function projectRequestContract(projected: RequestContractDetails, serialized: string): string { const recovery = projected.recovery; const next = recovery.nextRequest ? "\nCopy the nested recovery.nextRequest object unchanged; do not repeat the original query." - : `\nRecovery action: ${recovery.action}. ${recovery.reason}`; + : `\nRecovery action: ${recovery.action}.`; // Put the machine payload first so simple host adapters that locate the // first `nextRequest` marker see the exact nested request object. - return `\nError details: ${serialized}\n${prefix}${next}`; + return `\nsift-light failed: request rejected [${projected.code}].\nError details: ${serialized}${next}`; } /** Build the one bounded contract projection consumed by structured and text hosts. */ @@ -94,11 +89,10 @@ export function requestContractProjection( ): RequestContractProjection { const details = boundedRequestContractDetails(error.details); const serializedDetails = JSON.stringify(details); - const boundedMessage = error.message.toWellFormed().slice(0, 1_024); - const result = projectRequestContract(details, serializedDetails, boundedMessage); + const result = projectRequestContract(details, serializedDetails); if (Buffer.byteLength(result) <= MAX_REQUEST_RECOVERY_BYTES) return { details, text: result }; const compact: RequestContractDetails = { - code: boundedMessage.length > 0 ? details.code : "E_REQUEST_CONTRACT_PAYLOAD", + code: details.code, ...(details.mode ? { mode: details.mode } : {}), issues: [ { @@ -111,10 +105,6 @@ export function requestContractProjection( reason: "Exact recovery was omitted; correct the request explicitly.", }, }; - const compactText = projectRequestContract( - compact, - JSON.stringify(compact), - "The request-contract error exceeded the bounded payload budget; correct the request explicitly.", - ); + const compactText = projectRequestContract(compact, JSON.stringify(compact)); return { details: compact, text: compactText }; } diff --git a/src/request-contract.ts b/src/request-contract.ts index ed1a3ff2..13db31f1 100644 --- a/src/request-contract.ts +++ b/src/request-contract.ts @@ -115,6 +115,23 @@ function schemaError( ); } +function plainFilePatternRecovery( + mode: SiftLightMode, + input: Record, + invalid: readonly string[], +): boolean { + return ( + mode === "files" && + invalid.includes("pattern") && + input.query === undefined && + typeof input.pattern === "string" && + input.pattern.trim().length > 0 && + input.pattern.length <= 256 && + input.pattern.isWellFormed() && + !/[\\^$.*+?()[\]{}|\r\n\0]/u.test(input.pattern) + ); +} + function safeNextRequest( input: Record, mode: SiftLightMode, @@ -122,9 +139,12 @@ function safeNextRequest( ): Record | undefined { if (input.redact === true && containsSensitiveText(input)) return undefined; const safe = new Set(SAFE_DROP_FIELDS[mode] ?? []); + const plainFilePattern = plainFilePatternRecovery(mode, input, invalid); + if (plainFilePattern) safe.add("pattern"); if (invalid.some((field) => !safe.has(field))) return undefined; const next: Record = { ...input }; for (const field of invalid) delete next[field]; + if (plainFilePattern) next.query = input.pattern; if (selectorIssuesFor(next, mode).length > 0) return undefined; // A copied request must remain within the transport's bounded line budget. try { @@ -165,7 +185,14 @@ function fieldsError( // the signature of arguments shaped as a per-mode object, which the flat // schema never advertises. Naming the accepted fields replaces an otherwise // puzzling rejection with the rule and the valid names. - const flatRule = `Fields are flat: pass them at the top level, not inside a per-mode object. mode=${mode} accepts: ${modeFields(mode).join(", ")}.`; + const nestedModeObject = invalid.includes(mode); + const flatRule = nestedModeObject + ? `Fields are flat: pass them at the top level, not inside a per-mode object. mode=${mode} accepts: ${modeFields(mode).join(", ")}.` + : ""; + const filesPatternRule = + mode === "files" && invalid.includes("pattern") + ? "For filename or path discovery, put the literal name text in query; pattern is a content-search regex, so check any regex syntax before copying it." + : ""; return new RequestContractError( { code: "E_MODE_FIELDS", @@ -174,17 +201,26 @@ function fieldsError( recovery: nextRequest ? { action: "retry", - reason: `Remove only ${visibleFields} and copy the exact nextRequest; all other fields are preserved.`, + reason: plainFilePatternRecovery(mode, input, invalid) + ? "For filename discovery, move the plain text from pattern to query and copy nextRequest; all filters are preserved." + : `Remove only ${visibleFields} and copy the exact nextRequest; all other fields are preserved.`, nextRequest, } : { action: "manual", - reason: `${reason}; choose the mode explicitly or remove the fields yourself without changing the requested scope. ${flatRule}`, + reason: [ + reason, + filesPatternRule || + "Choose the mode explicitly or remove unsupported fields without changing the requested scope.", + flatRule, + ] + .filter(Boolean) + .join(" "), }, }, nextRequest - ? `${reason}; retry the exact nextRequest without repeating the original query.` - : `${reason}; no semantics-preserving automatic request is available. ${flatRule} Preserve valid path, filters, redact and cursor fields when choosing the next request.`, + ? `${reason}; ${plainFilePatternRecovery(mode, input, invalid) ? "move plain filename text to query" : "retry the exact nextRequest"}.` + : `${reason}; no semantics-preserving automatic request is available. ${filesPatternRule || flatRule || "Check the mode's accepted fields."}`, ); } diff --git a/src/semantic-judge.ts b/src/semantic-judge.ts index 982b8f2e..b3b7bb75 100644 --- a/src/semantic-judge.ts +++ b/src/semantic-judge.ts @@ -134,8 +134,7 @@ function requestBody( for (const candidate of candidates) { questions[candidate.id] = { type: "choice", - instructions: - "Classify the candidate by what it actually does for the requested behavior. Judge the code excerpt, not just matching words.", + instructions: `Classify only candidate ${candidate.id} in state.candidates for state.query. Judge that candidate's excerpt by its actual behavior, not by matching words or the other candidates.`, criteria: { "implementation-candidate": "The excerpt appears to implement the requested behavior or its core decision/side effect.", From 16caaaf695211c175ad136d6677b0e114a926931 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=9D=E5=84=BF?= <274762368+xcjy8bao@users.noreply.github.com> Date: Wed, 23 Sep 2026 13:07:03 +0800 Subject: [PATCH 2/2] fix: make vector search opt-in across hosts --- CHANGELOG.md | 4 + README.md | 7 +- README.zh-CN.md | 7 +- plugins/sift-light/omp-extension.mjs | 35 ++- src/config-reader.ts | 14 +- src/index.ts | 14 +- src/mcp-semantic-judge.ts | 27 +- src/mcp-server.mjs | 366 ++++++++++++++------------- src/mcp-server.ts | 24 +- src/mcp.ts | 2 + src/omp-index.ts | 10 +- src/prompt-guidelines.ts | 10 +- src/semantic-judge.ts | 17 +- src/service.ts | 8 + src/tool-schema.ts | 6 +- 15 files changed, 317 insertions(+), 234 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bd1dcd8..fe753f8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## 1.0.1 — 2026-09-23 +### Changed + +- Local vector search is now off by default. Existing `concept` and `hybrid` calls require `"vectorSearchEnabled": true` in the active `sift-light.json` and an installed model; otherwise they fail explicitly. Ordinary search remains available. Installing the model or enabling the optional Jev judge alone does not opt in. + ### Fixed - Improve semantic ranking when very short passages compete with more detailed source evidence. Raw cosine similarity remains available alongside the ranking score; results are still relevance candidates, not verified behavior. diff --git a/README.md b/README.md index f4fc0c5e..a5ffb71e 100644 --- a/README.md +++ b/README.md @@ -24,13 +24,17 @@ Looking for an error message, a sentence or a name is like giving a librarian a ### Recover when the remembered wording is not exact +For routine development work, searches with a known name, symbol, filename or error text use the fast exact path by default; omitting `mode` does not load the local embedding model. Vector search is disabled by default for Pi, OMP and MCP clients. `concept` and `hybrid` fail clearly while it is disabled; neither silently becomes an exact-only search. This behavior is built into the plugin and does not depend on a personal `AGENTS.md`. An uncached semantic search can take tens of seconds, so it should not be part of every code lookup. + +To opt in, set `"vectorSearchEnabled": true` in the active `sift-light.json`, install the local model with `npx -y --package sift-light@latest sift-light-model --install-model`, and restart the host. Pi and OMP use the configuration paths below; MCP needs `SIFT_LIGHT_CONFIG` set to that file in its server environment. Installing the model alone does not enable vector search. `semanticJudge.enabled` is a separate option for remote candidate classification: it does not enable vector search and remains inactive while vector search is disabled. + Use `mode: "hybrid"` with one natural-language `query` when a sentence may have been remembered with different wording. Hybrid always runs an exact literal search and the installed local Concept model under one owned request. Exact evidence appears first; semantic candidates are clearly labeled as relevance candidates and removed when they overlap an exact match. Semantic candidates retain raw cosine scores and use a bounded short-passage correction for ranking; both scores remain visible in structured details and neither proves runtime behavior. The initial page shares counts, coverage, source references, one inspection cursor and a compact preview instead of concatenating two complete responses. `conceptLimit` changes only the non-overlapping semantic supplement (default 3, maximum 20); it never displaces literal evidence. The returned matches request opens the same snapshot's complete exact-first pagination without rerunning either search. Concept ranking covers every UTF-8 passage admitted by the request's documented source budget; it no longer samples a fixed prefix of the scope. Concept and hybrid searches automatically admit up to 2,000 files and process them sequentially in bounded 200-file batches, then merge every batch into one global ranking and one coverage result. Batches share the request's 32 MiB read budget, so raising the file ceiling does not multiply the content budget. Users do not need to plan or continue batches themselves. `maxFilesToParse` remains an optional advanced hard ceiling when a smaller scope is intentional. Passages that exceed the model token window are ranked through overlapping token-safe windows, so later text is not silently discarded. Offline embeddings are cached by content, model revision and chunking revision in a bounded 512 MiB local cache. Repeated content is reused, changed content misses naturally, and cache write or cleanup failures remain visible in the result. Slow Concept and hybrid requests return within the default five-second wait window with `status: "waiting"` or `"running"`, an `operationId`, progress and an exact `nextRequest` such as `{ "mode": "await", "operationId": "..." }`. Copy that request unchanged: it resumes the same computation and never restarts the query or downgrades to a literal-only result. A final result remains available for stable re-fetch for 10 minutes, with up to 32 terminal results retained per service session, and `mode: "cancel"` stops the owned work and waits for cleanup. Each service session admits at most eight pending operations; an operation has one total deadline controlled by `SIFT_LIGHT_CONCEPT_TIMEOUT_MS` (integer milliseconds from 1000 through 3600000; default 600000) and a 120-second idle continuation lease. A real model, source or resource failure is returned as a failure with its diagnostic. Source generation is re-enumerated and re-verified before publication, so changes refresh the operation and mixed versions are never marked complete. Admission planning counts (`filesEnumerated`, `filesAdmitted`, `filesSkippedEmpty`, `filesUnavailable`, `passagesQueued`, `batchesPlanned`, `batchesCompleted`) stay visible. Empty files are a normal skip and do not mark the result partial. -The first uncached Concept or hybrid search loads the local model and may take tens of seconds and more than 1 GiB of inference-worker memory, depending on the machine and search scope. Cached searches avoid most inference work. These are workload-dependent observations, not a latency or memory guarantee. If the result is waiting during `model-loading`, follow its `nextRequest` to continue the same operation. +The first uncached Concept or hybrid search loads the local model and may take tens of seconds and more than 1 GiB of inference-worker memory, depending on the machine and search scope. Later searches reuse cached passage embeddings even when the query wording changes, although a new query still loads the model briefly to embed that query. Changed passages are recomputed; cache eviction, deletion or broad source changes can cause another cold run. Exact search modes do not load the model. These are workload-dependent observations, not a latency or memory guarantee. If the result is waiting during `model-loading`, follow its `nextRequest` to continue the same operation. ### Give several search conditions together @@ -122,6 +126,7 @@ The feature is enabled only when `semanticJudge.enabled` is explicitly set to `t { "locale": "en", "enforceSearch": "hard", + "vectorSearchEnabled": false, "semanticJudge": { "enabled": false, "provider": "jev", diff --git a/README.zh-CN.md b/README.zh-CN.md index 51cd4093..8e4e2181 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -24,13 +24,17 @@ ### 记得不完全准确，也能一次找回 +日常开发时，已知名称、符号、文件名或报错文本的搜索默认走快速的精确路径；省略 `mode` 不会加载本地向量模型。Pi、OMP 和 MCP 的向量搜索默认关闭；关闭时 `concept` 和 `hybrid` 会明确报错，不会悄悄降级成只有精确结果。这是插件自身的行为，不依赖个人的 `AGENTS.md`。未缓存的语义搜索可能耗时数十秒，不应成为每次查代码的默认成本。 + +需要时，在当前生效的 `sift-light.json` 中设置 `"vectorSearchEnabled": true`，执行 `npx -y --package sift-light@latest sift-light-model --install-model` 安装本地模型，然后重启宿主。Pi 和 OMP 的配置路径见下文；MCP 需要在服务进程环境中用 `SIFT_LIGHT_CONFIG` 指向该文件。只安装模型不会自动启用向量搜索。`semanticJudge.enabled` 是远程候选分类的独立开关，不会启用向量搜索；向量关闭时它也不会运行。 + 当一句话可能记错了措辞时，可以用一个自然语言 `query` 调用 `mode: "hybrid"`。Hybrid 会在同一个受控请求中始终执行精确字面搜索和已安装的本地 Concept 模型：精确证据固定排在前面，语义候选明确标注且只表示相关性候选，与精确命中范围重叠的候选会被去重。语义候选保留原始余弦分数，并用有界短段修正分数排序；两种分数均在结构化详情中可见，不能当作运行时证明。初始页面共享计数、覆盖状态、来源引用和一个检查游标，以紧凑预览代替拼接两份完整响应。`conceptLimit` 只调整不重叠的语义补充数量（默认 3，最大 20），不会挤占字面证据；返回的 matches 请求从同一个快照开始完整的精确优先分页，不会重新执行任一搜索。 Concept 排名会覆盖请求所声明源码预算内接纳的全部 UTF-8 段落，不再固定抽取范围开头的一小部分。Concept 和 hybrid 默认会自动接纳最多 2,000 个文件，在内部按每批 200 个文件顺序处理，再合并成一次全局排名和一份覆盖结果。所有批次共享同一个请求的 32 MiB 读取预算，扩大文件上限不会把内容预算成倍放大。用户不需要自己计算或续接批次；只有确实想主动缩小范围时，才需要把 `maxFilesToParse` 作为可选的高级硬上限。超过模型 token 窗口的段落会拆成带重叠、且保证不截断的窗口参与排名，后半段内容不会被静默丢弃。离线 embedding 按内容、模型版本和分段版本缓存在本地，缓存上限为 512 MiB；重复内容直接复用，内容变化自然失效，缓存写入或清理失败会在结果中明确显示。 Concept 或 hybrid 较慢时，会在默认五秒等待窗口内返回 `status: "waiting"` 或 `"running"`、`operationId`、进度和精确的 `nextRequest`，例如 `{ "mode": "await", "operationId": "..." }`。请原样复制这个请求：它会续接同一个计算，不会重启查询，也不会降级成只有字面的结果。最终结果可稳定复取十分钟；每个服务会话最多保留 32 个终态结果。`mode: "cancel"` 会停止自有任务并等待清理完成。每个服务会话最多同时接纳八个 pending operation；单个 operation 使用 `SIFT_LIGHT_CONCEPT_TIMEOUT_MS` 指定一个总执行时限（整数毫秒，1000–3600000，默认 600000），另有 120 秒无人续接租期。真实模型、来源或资源故障会以明确失败返回。发布结果前会重新枚举并校验同一来源 generation；源文件变化会刷新 operation，混合版本不会被标成 complete。接纳计划计数（`filesEnumerated`、`filesAdmitted`、`filesSkippedEmpty`、`filesUnavailable`、`passagesQueued`、`batchesPlanned`、`batchesCompleted`）会在结果里明确显示。空文件属于正常跳过，不会把结果标成 partial。 -首次运行尚未缓存的 Concept 或 hybrid 搜索时需要加载本地模型；耗时可能达到数十秒，推理 worker 内存也可能超过 1 GiB，具体取决于机器和搜索范围。缓存热后可省去大部分推理工作。这只是随负载变化的观察，不是延迟或内存保证。结果在 `model-loading` 阶段等待时，请按返回的 `nextRequest` 续接同一个 operation。 +首次运行尚未缓存的 Concept 或 hybrid 搜索时需要加载本地模型；耗时可能达到数十秒，推理 worker 内存也可能超过 1 GiB，具体取决于机器和搜索范围。后续搜索即使换了问题，也会复用已缓存的段落向量；新问题仍会短暂加载模型来计算问题向量。修改过的段落需要重算；缓存被清理、淘汰或大量源码变化时可能再次冷运行。精确搜索模式不会加载模型。这些都是随负载变化的观察，不是延迟或内存保证。结果在 `model-loading` 阶段等待时，请按返回的 `nextRequest` 续接同一个 operation。 ### 几个条件，可以一起交代 @@ -122,6 +126,7 @@ Hybrid 搜索默认只使用本地能力。可选的语义判断器可以对保 { "locale": "zh-CN", "enforceSearch": "hard", + "vectorSearchEnabled": false, "semanticJudge": { "enabled": false, "provider": "jev", diff --git a/plugins/sift-light/omp-extension.mjs b/plugins/sift-light/omp-extension.mjs index 713b35a5..f69b45a4 100644 --- a/plugins/sift-light/omp-extension.mjs +++ b/plugins/sift-light/omp-extension.mjs @@ -71,6 +71,7 @@ var DEFAULT_SEMANTIC_JUDGE_CONFIG = { var DEFAULT_SIFT_LIGHT_CONFIG = { locale: "en", enforceSearch: "hard", + vectorSearchEnabled: false, semanticJudge: DEFAULT_SEMANTIC_JUDGE_CONFIG }; function resolveSiftLightConfigPath(agentDirectory, environment = process.env) { @@ -163,18 +164,22 @@ function parseConfig(value, path) { if (!isRawSiftLightConfig(value)) { throw new Error(`Invalid sift-light config at ${path}: expected a JSON object`); } - const unknown = Object.keys(value).filter((key) => !["locale", "enforceSearch", "semanticJudge"].includes(key)); + const unknown = Object.keys(value).filter((key) => !["locale", "enforceSearch", "vectorSearchEnabled", "semanticJudge"].includes(key)); if (unknown.length > 0) { - throw new Error(`Invalid sift-light config at ${path}: unsupported configuration fields; only locale, enforceSearch and semanticJudge are accepted`); + throw new Error(`Invalid sift-light config at ${path}: unsupported configuration fields; only locale, enforceSearch, vectorSearchEnabled and semanticJudge are accepted`); } - const { locale, enforceSearch } = value; + const { locale, enforceSearch, vectorSearchEnabled } = value; if (locale !== undefined && locale !== "en" && locale !== "zh-CN") { throw new Error(`Invalid sift-light config at ${path}: locale must be "en" or "zh-CN"`); } + if (vectorSearchEnabled !== undefined && typeof vectorSearchEnabled !== "boolean") { + throw new Error(`Invalid sift-light config at ${path}: vectorSearchEnabled must be a boolean`); + } const enforcement = normalizeSearchEnforcement(enforceSearch, `config at ${path}`); return { locale: locale ?? DEFAULT_SIFT_LIGHT_CONFIG.locale, enforceSearch: enforcement, + vectorSearchEnabled: vectorSearchEnabled ?? DEFAULT_SIFT_LIGHT_CONFIG.vectorSearchEnabled ?? false, semanticJudge: parseSemanticJudge(value.semanticJudge, path) }; } @@ -9482,6 +9487,10 @@ function createSemanticJudgeIntegration(config, environment = process.env, fetch } return { config, runner: createJevRunner(config, key, fetcher) }; } +function createConfiguredSemanticJudgeIntegration(config, environment = process.env) { + const judge = config.semanticJudge ?? DEFAULT_SEMANTIC_JUDGE_CONFIG; + return config.vectorSearchEnabled === true ? createSemanticJudgeIntegration(judge, environment) : createDisabledSemanticJudgeIntegration(judge); +} function baseDetails(config) { return { enabled: config.enabled, @@ -12770,6 +12779,7 @@ class SiftLightService { #runRipgrep; #snapshots; #summaryFileLimit; + #vectorSearchEnabled; #capabilities = new LanguageCapabilityCatalog; #evidence; #lifecycle = new AbortController; @@ -12780,12 +12790,16 @@ class SiftLightService { this.#runRipgrep = options.runRipgrep; this.#snapshots = options.snapshots ?? new SnapshotStore; this.#summaryFileLimit = options.summaryFileLimit ?? DEFAULT_SUMMARY_FILE_LIMIT; + this.#vectorSearchEnabled = options.vectorSearchEnabled ?? options.conceptSearch !== undefined; this.#operations = new OperationLifecycle({ deadlineMs: resolveConceptTimeoutMs() }); this.#evidence = new EvidenceService(this.#runRipgrep, this.#snapshots, options.structure, options.conceptSearch, options.semanticJudge); } async search(input, cwd, signal, options = {}) { validateRawSearchInput(input); validateRequestContract(input); + if (!this.#vectorSearchEnabled && (input.mode === "concept" || input.mode === "hybrid")) { + throw new SiftLightError(`${input.mode} search is disabled; set vectorSearchEnabled to true in sift-light.json and restart the host`); + } let request; if (input.mode === "await" || input.mode === "cancel") { request = this.#operationCommand(input, cwd, signal); @@ -13188,7 +13202,7 @@ ${page.body}${rangeNote}${contextNote}${missingSelectionNote} var SOURCE_OUTPUT_GUIDANCE = "Auto/summary text may include bounded source excerpts; ordinary matches text is metadata-only. Inspect may return bounded source windows covering an entire small file. Analysis text may include semantic passages; structured details may retain excerpts, names and signatures. Follow output limits, coverage and continuations."; function siftLightPromptGuidelines(structuredOutput = true) { return [ - `Use sift-light for read-only content search. ${SOURCE_OUTPUT_GUIDANCE} Omit mode and limit for automatic detail/summary selection; use mode="matches" for ordinary match metadata.`, + `Use sift-light for read-only content search. ${SOURCE_OUTPUT_GUIDANCE} For routine development searches, start with fast exact content, filename or applicable structural modes when the request has a usable name, symbol, error text or other literal clue. Omitted mode is ordinary exact search and never loads the local embedding model. Vector search is disabled by default; concept/hybrid require vectorSearchEnabled:true in sift-light.json and an installed model. They can take tens of seconds on an uncached scope, so select them only when semantic recall is needed. Omit mode and limit for automatic detail/summary selection; use mode="matches" for ordinary match metadata.`, `An omitted path searches the project cwd. Use scope:"strict" for a question restricted to one path; otherwise, if an explicit subpath has zero matches, ordinary and content-analysis searches retry from cwd and return project-wide counts with an expansion notice. Explicit absolute paths and .. traversal can search outside cwd, except protected external system areas and .git internals. Git changes mode remains cwd-scoped.`, `Search output includes counts, categories, ranked paths, coverage and continuation metadata. Source excerpts may contain the searched text. Use mode="inspect" or the host read capability when exact source is required for an edit or verification.`, `Use file and directory distributions to choose evidence. Reuse the visible cursor with path or paths for match metadata; mode="summary" pages the remaining file statistics. Match counts are not relevance scores.`, @@ -13197,7 +13211,7 @@ function siftLightPromptGuidelines(structuredOutput = true) { `Use anyOf:["term1","term2"] when every exact occurrence of 2-64 literals is needed in one version-bound result. It is case-sensitive, reports anonymized condition counts, and runs requests above eight terms as bounded parallel chunks. Large condition inventories have separate continuation pages; copy those requests to retrieve the complete counts.`, `For a changed-code question, add changes:{base:"HEAD",scope:"lines",side:"new"}; omit target for the working tree, use side:"old" for deleted-side statistics. Copy returned continuation requests to preserve source versions.`, `Use mode:"capabilities" when the language or requested operation is unclear to get a compact lazy inventory. Use mode:"outline" with a concrete source file path for symbol counts and locations, mode:"imports" for static relationships, and mode:"tests" for related-test candidates. Their text pages summarize metadata; structured details can retain source evidence.`, - `Use mode:"files" plus query for unknown filenames and fuzzy paths. Multi-word filename queries require each word literally in the path; business concepts belong in hybrid/concept. Use wholeWord:true for a single-pattern whole-word search. exclude contains file globs, not content negation.`, + `Use mode:"files" plus query for unknown filenames and fuzzy paths. Multi-word filename queries require each word literally in the path; use hybrid/concept only for business concepts that cannot be located by a literal clue. Use wholeWord:true for a single-pattern whole-word search. exclude contains file globs, not content negation.`, `Use mode:"structure" only for JS/TS/TSX/Go, with a required nonempty ast-grep pattern such as "compare($X, $X)" or "send()" for code shapes across whitespace; the text page reports structural counts and locations; structured details can retain matched source evidence.`, `Use mode:"concept" plus a natural-language query when names are unknown. It runs a pinned local multilingual model only after explicit installation; no search downloads weights or sends code to a remote model. Results expose candidate counts, score statistics, paths and the bounded ranked passage behind each candidate. Similarity scores identify candidates, not correctness.`, `Use mode:"hybrid" plus query when wording may differ from the source. It reports exact and semantic counts separately, removes overlap, and retains one pageable evidence snapshot. conceptLimit changes only the semantic candidate count; retained candidates keep their bounded source passages.`, @@ -20553,12 +20567,12 @@ function stringEnum(values, options) { ...options?.description ? { description: options.description } : {} }); } -var SIFT_LIGHT_DESCRIPTION = `Search and navigate code with bounded, verifiable evidence. Ordinary pattern searches use auto detail/summary; pattern is regex by default and literal=true matches source text exactly. A path selects an existing exact file or root; use mode=files with query to discover an unknown name. scope=strict prevents zero-result path expansion and wholeWord requires word boundaries. mode=capabilities returns a compact names-only project language inventory and the modes available for each detected language; capability providers are loaded only when the requested analysis runs. It never starts a parser, compiler, model or language server. mode=concept accepts a natural-language query, path and source filters; mode=hybrid uses one natural-language query for exact and local concept evidence, ranks exact evidence first, and retains a bounded semantic supplement. An explicitly enabled semantic judge may classify hybrid candidates, but it is disabled by default and never turns classification into a runtime proof. Slow concept/hybrid requests return status=waiting or running with operationId, progress, and an exact nextRequest using mode=await; copy that request unchanged to continue the same computation. Await expiry never downgrades evidence to literal-only or partial, and final results remain stable for the operation retention window. mode=cancel explicitly stops one operation. allOf and anyOf are explicit literal variants and cannot be mixed with pattern/literal; limit and context are output intent and are never silently dropped. modifiedAfter/modifiedBefore filter worktree files by inclusive/exclusive modification-time bounds in Unix milliseconds. structure requires a nonempty AST pattern and JS/TS/TSX/Go sources; lang is not a field. Outline uses a concrete source file path (not a directory) or retained cursor+matchIndex and follows declared syntax capabilities. imports/tests return bounded static module and related-test candidates without proving runtime execution. validate checks saved source evidence against its recorded origin. Partial coverage stays explicit. ${REQUEST_USAGE_GUIDANCE}`; -var SIFT_LIGHT_MODEL_DESCRIPTION = `Bounded local evidence search. ${MODEL_USAGE_GUIDANCE}. Copy cursors; analysis is evidence, not proof.`; +var SIFT_LIGHT_DESCRIPTION = `Search and navigate code with bounded, verifiable evidence. Routine searches should use exact content, filenames or applicable structural modes first. Omitted mode is ordinary exact search and does not load the embedding model; concept/hybrid require vectorSearchEnabled:true in sift-light.json plus an installed model and may take tens of seconds on an uncached scope. Ordinary pattern searches use auto detail/summary; pattern is regex by default and literal=true matches source text exactly. A path selects an existing exact file or root; use mode=files with query to discover an unknown name. scope=strict prevents zero-result path expansion and wholeWord requires word boundaries. mode=capabilities returns a compact names-only project language inventory and the modes available for each detected language; capability providers are loaded only when the requested analysis runs. It never starts a parser, compiler, model or language server. mode=concept accepts a natural-language query, path and source filters; mode=hybrid uses one natural-language query for exact and local concept evidence, ranks exact evidence first, and retains a bounded semantic supplement. An explicitly enabled semantic judge may classify hybrid candidates, but it is disabled by default and never turns classification into a runtime proof. Slow concept/hybrid requests return status=waiting or running with operationId, progress, and an exact nextRequest using mode=await; copy that request unchanged to continue the same computation. Await expiry never downgrades evidence to literal-only or partial, and final results remain stable for the operation retention window. mode=cancel explicitly stops one operation. allOf and anyOf are explicit literal variants and cannot be mixed with pattern/literal; limit and context are output intent and are never silently dropped. modifiedAfter/modifiedBefore filter worktree files by inclusive/exclusive modification-time bounds in Unix milliseconds. structure requires a nonempty AST pattern and JS/TS/TSX/Go sources; lang is not a field. Outline uses a concrete source file path (not a directory) or retained cursor+matchIndex and follows declared syntax capabilities. imports/tests return bounded static module and related-test candidates without proving runtime execution. validate checks saved source evidence against its recorded origin. Partial coverage stays explicit. ${REQUEST_USAGE_GUIDANCE}`; +var SIFT_LIGHT_MODEL_DESCRIPTION = `Bounded local evidence search. Default exact search is model-free; concept/hybrid require vectorSearchEnabled:true in sift-light.json and a model. ${MODEL_USAGE_GUIDANCE}. Copy cursors; analysis is evidence, not proof.`; var siftLightSchema = _Object_({ query: Optional(String2({ maxLength: 256, - description: `${fieldGuidance("query")}. Hybrid uses the same query as exact literal text and as the local concept query. Discovery modes preserve their requested path. Concept and hybrid require an explicitly installed local model. A semantic judge is optional and remains disabled unless the active configuration explicitly enables it.` + description: `${fieldGuidance("query")}. Hybrid uses the same query as exact literal text and as the local concept query. Discovery modes preserve their requested path. Concept and hybrid require vectorSearchEnabled:true in sift-light.json and an explicitly installed local model. A semantic judge is optional and remains disabled unless the active configuration explicitly enables it.` })), scope: Optional(stringEnum(["strict", "expand"], { description: `${fieldGuidance("scope")}; expand (default) retries ordinary content search from project cwd. Applies to ordinary, multi-term and role searches.` @@ -20864,10 +20878,11 @@ async function registerOmpSiftLightExtension(pi, searchPolicyAssets = new URL(". const resolvedConfig = config ?? await readSiftLightConfigFile(resolveSiftLightConfigPath(ompAgentDir()), { missing: process.env[SIFT_LIGHT_CONFIG_ENV]?.trim() ? "error" : "defaults" }); - const semanticJudge = createSemanticJudgeIntegration(resolvedConfig.semanticJudge ?? DEFAULT_SEMANTIC_JUDGE_CONFIG); + const semanticJudge = createConfiguredSemanticJudgeIntegration(resolvedConfig); const runtime = new SiftLightRuntime(new SiftLightService({ runRipgrep: createRipgrepRunner(), structure: createCtagsStructureProvider(), + vectorSearchEnabled: resolvedConfig.vectorSearchEnabled === true, semanticJudge })); const { locale } = resolvedConfig; @@ -20886,7 +20901,7 @@ async function registerOmpSiftLightExtension(pi, searchPolicyAssets = new URL(". pi.registerTool({ name: SIFT_LIGHT_LABEL, label: SIFT_LIGHT_LABEL, - description: "Search and navigate code with bounded, verifiable evidence. Use pattern for content or mode=files with query for filenames.", + description: "Search code with bounded evidence. Routine searches use pattern for fast exact content or mode=files with query for filenames; concept/hybrid require vectorSearchEnabled:true in sift-light.json.", approval: "read", promptSnippet: "Search file contents without flooding context", promptGuidelines: siftLightPromptGuidelines(), diff --git a/src/config-reader.ts b/src/config-reader.ts index 745ab489..c3724cb3 100644 --- a/src/config-reader.ts +++ b/src/config-reader.ts @@ -12,6 +12,7 @@ export const SIFT_LIGHT_ENFORCEMENT_ENV = "SIFT_LIGHT_ENFORCE_SEARCH"; export interface SiftLightConfig { locale: SiftLightLocale; enforceSearch?: SearchEnforcementMode; + vectorSearchEnabled?: boolean; semanticJudge?: SemanticJudgeConfig; } @@ -44,12 +45,14 @@ export const DEFAULT_SEMANTIC_JUDGE_CONFIG: Readonly = { export const DEFAULT_SIFT_LIGHT_CONFIG: Readonly = { locale: "en", enforceSearch: "hard", + vectorSearchEnabled: false, semanticJudge: DEFAULT_SEMANTIC_JUDGE_CONFIG, }; interface RawSiftLightConfig { locale?: unknown; enforceSearch?: unknown; + vectorSearchEnabled?: unknown; semanticJudge?: unknown; } @@ -221,21 +224,26 @@ function parseConfig(value: unknown, path: string): SiftLightConfig { throw new Error(`Invalid sift-light config at ${path}: expected a JSON object`); } const unknown = Object.keys(value).filter( - (key) => !["locale", "enforceSearch", "semanticJudge"].includes(key), + (key) => !["locale", "enforceSearch", "vectorSearchEnabled", "semanticJudge"].includes(key), ); if (unknown.length > 0) { throw new Error( - `Invalid sift-light config at ${path}: unsupported configuration fields; only locale, enforceSearch and semanticJudge are accepted`, + `Invalid sift-light config at ${path}: unsupported configuration fields; only locale, enforceSearch, vectorSearchEnabled and semanticJudge are accepted`, ); } - const { locale, enforceSearch } = value; + const { locale, enforceSearch, vectorSearchEnabled } = value; if (locale !== undefined && locale !== "en" && locale !== "zh-CN") { throw new Error(`Invalid sift-light config at ${path}: locale must be "en" or "zh-CN"`); } + if (vectorSearchEnabled !== undefined && typeof vectorSearchEnabled !== "boolean") { + throw new Error(`Invalid sift-light config at ${path}: vectorSearchEnabled must be a boolean`); + } const enforcement = normalizeSearchEnforcement(enforceSearch, `config at ${path}`); return { locale: locale ?? DEFAULT_SIFT_LIGHT_CONFIG.locale, enforceSearch: enforcement, + vectorSearchEnabled: + vectorSearchEnabled ?? DEFAULT_SIFT_LIGHT_CONFIG.vectorSearchEnabled ?? false, semanticJudge: parseSemanticJudge(value.semanticJudge, path), }; } diff --git a/src/index.ts b/src/index.ts index 52be1a01..597d1279 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,11 +1,6 @@ import { SIFT_LIGHT_DESCRIPTION, siftLightSchema } from "./tool-schema.js"; import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; -import { - normalizeSearchEnforcement, - readSiftLightConfig, - DEFAULT_SEMANTIC_JUDGE_CONFIG, - type SiftLightConfig, -} from "./config.js"; +import { normalizeSearchEnforcement, readSiftLightConfig, type SiftLightConfig } from "./config.js"; import { resolveContextBudget } from "./context-budget.js"; import { createRipgrepRunner } from "./rg.js"; import { createCtagsStructureProvider } from "./structure.js"; @@ -17,7 +12,7 @@ import type { SiftLightDetails } from "./types.js"; import { renderSiftLightCall, renderSiftLightResult } from "./tui/renderers.js"; import { registerPiSearchPolicy } from "./pi-search-policy.js"; import { modelErrorText } from "./model-error.js"; -import { createSemanticJudgeIntegration } from "./semantic-judge.js"; +import { createConfiguredSemanticJudgeIntegration } from "./semantic-judge.js"; const SIFT_LIGHT_LABEL = "sift-light"; @@ -27,13 +22,12 @@ export async function registerSiftLightExtension( pi: ExtensionAPI, config: SiftLightConfig, ): Promise { - const semanticJudge = createSemanticJudgeIntegration( - config.semanticJudge ?? DEFAULT_SEMANTIC_JUDGE_CONFIG, - ); + const semanticJudge = createConfiguredSemanticJudgeIntegration(config); const runtime = new SiftLightRuntime( new SiftLightService({ runRipgrep: createRipgrepRunner(), structure: createCtagsStructureProvider(), + vectorSearchEnabled: config.vectorSearchEnabled === true, semanticJudge, }), ); diff --git a/src/mcp-semantic-judge.ts b/src/mcp-semantic-judge.ts index b42a6e63..244cf761 100644 --- a/src/mcp-semantic-judge.ts +++ b/src/mcp-semantic-judge.ts @@ -1,11 +1,12 @@ import { + DEFAULT_SIFT_LIGHT_CONFIG, DEFAULT_SEMANTIC_JUDGE_CONFIG, readSiftLightConfigFile, SIFT_LIGHT_CONFIG_ENV, } from "./config-reader.js"; import { createDisabledSemanticJudgeIntegration, - createSemanticJudgeIntegration, + createConfiguredSemanticJudgeIntegration, type SemanticJudgeIntegration, } from "./semantic-judge.js"; @@ -19,17 +20,27 @@ function configuredPath(environment: NodeJS.ProcessEnv): string | undefined { * The MCP process must receive the same config path explicitly in its own host * environment; an absent path remains an observable, local-only disabled state. */ -export async function createMcpSemanticJudgeIntegration( +export async function createMcpSearchFeatures( environment: NodeJS.ProcessEnv = process.env, -): Promise { +): Promise<{ semanticJudge: SemanticJudgeIntegration; vectorSearchEnabled: boolean }> { const path = configuredPath(environment); - if (!path) return createDisabledSemanticJudgeIntegration(DEFAULT_SEMANTIC_JUDGE_CONFIG); + if (!path) + return { + semanticJudge: createDisabledSemanticJudgeIntegration(DEFAULT_SEMANTIC_JUDGE_CONFIG), + vectorSearchEnabled: DEFAULT_SIFT_LIGHT_CONFIG.vectorSearchEnabled === true, + }; const config = await readSiftLightConfigFile(path, { missing: "error" }); - return createSemanticJudgeIntegration( - config.semanticJudge ?? DEFAULT_SEMANTIC_JUDGE_CONFIG, - environment, - ); + return { + semanticJudge: createConfiguredSemanticJudgeIntegration(config, environment), + vectorSearchEnabled: config.vectorSearchEnabled === true, + }; +} + +export async function createMcpSemanticJudgeIntegration( + environment: NodeJS.ProcessEnv = process.env, +): Promise { + return (await createMcpSearchFeatures(environment)).semanticJudge; } export function mcpSemanticJudgeConfigSource( diff --git a/src/mcp-server.mjs b/src/mcp-server.mjs index 6d56aa56..fbad10ef 100755 --- a/src/mcp-server.mjs +++ b/src/mcp-server.mjs @@ -9562,6 +9562,160 @@ function parsePythonOutline(document) { // src/hybrid-search.ts import { resolve as resolve20 } from "node:path"; +// src/config-reader.ts +import { readFile as readFile2 } from "node:fs/promises"; +var SIFT_LIGHT_CONFIG_ENV = "SIFT_LIGHT_CONFIG"; +var SEMANTIC_JUDGE_API_KEY_ENVS = ["TYPESAFE_API_KEY", "SIFT_LIGHT_JEV_API_KEY"]; +var DEFAULT_SEMANTIC_JUDGE_CONFIG = { + enabled: false, + provider: "jev", + endpoint: "https://api.typesafe.ai/v1/systemone", + apiKeyEnv: SEMANTIC_JUDGE_API_KEY_ENVS[0], + model: "jev-latest", + timeoutMs: 120000, + maxCandidates: 20, + maxRetries: 2 +}; +var DEFAULT_SIFT_LIGHT_CONFIG = { + locale: "en", + enforceSearch: "hard", + vectorSearchEnabled: false, + semanticJudge: DEFAULT_SEMANTIC_JUDGE_CONFIG +}; +function hasErrorCode(error, codes) { + return error instanceof Error && "code" in error && codes.includes(String(error.code)); +} +function isMissingFile(error) { + return hasErrorCode(error, ["ENOENT"]); +} +function isRawSiftLightConfig(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function isRawSemanticJudgeConfig(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function boundedInteger2(value, fallback, minimum, maximum, field) { + const candidate = value ?? fallback; + if (typeof candidate !== "number" || !Number.isSafeInteger(candidate) || candidate < minimum || candidate > maximum) { + throw new Error(`Invalid sift-light ${field}: expected an integer from ${String(minimum)} through ${String(maximum)}`); + } + return candidate; +} +function parseSemanticJudge(value, path) { + if (value === undefined) + return { ...DEFAULT_SEMANTIC_JUDGE_CONFIG }; + if (!isRawSemanticJudgeConfig(value)) { + throw new Error(`Invalid sift-light config at ${path}: semanticJudge must be an object`); + } + const unknown = Object.keys(value).filter((key) => ![ + "enabled", + "provider", + "endpoint", + "apiKeyEnv", + "model", + "timeoutMs", + "maxCandidates", + "maxRetries" + ].includes(key)); + if (unknown.length > 0) { + throw new Error(`Invalid sift-light config at ${path}: unsupported semanticJudge fields; accepted fields are enabled, provider, endpoint, apiKeyEnv, model, timeoutMs, maxCandidates and maxRetries`); + } + const enabled = value.enabled ?? DEFAULT_SEMANTIC_JUDGE_CONFIG.enabled; + if (typeof enabled !== "boolean") { + throw new Error(`Invalid sift-light config at ${path}: semanticJudge.enabled must be a boolean`); + } + const provider = value.provider ?? DEFAULT_SEMANTIC_JUDGE_CONFIG.provider; + if (provider !== "jev") { + throw new Error(`Invalid sift-light config at ${path}: semanticJudge.provider must be "jev"`); + } + const endpoint = value.endpoint ?? DEFAULT_SEMANTIC_JUDGE_CONFIG.endpoint; + if (typeof endpoint !== "string" || endpoint.length === 0 || endpoint.length > 2048) { + throw new Error(`Invalid sift-light config at ${path}: semanticJudge.endpoint must be a nonempty URL`); + } + let parsedEndpoint; + try { + parsedEndpoint = new URL(endpoint); + } catch (error) { + throw new Error(`Invalid sift-light config at ${path}: semanticJudge.endpoint must be a URL`, { + cause: error + }); + } + const loopbackHosts = new Set(["localhost", "127.0.0.1", "[::1]"]); + const secureEndpoint = parsedEndpoint.protocol === "https:" || parsedEndpoint.protocol === "http:" && loopbackHosts.has(parsedEndpoint.hostname); + if (!secureEndpoint || parsedEndpoint.username || parsedEndpoint.password) { + throw new Error(`Invalid sift-light config at ${path}: semanticJudge.endpoint must use HTTPS, except that HTTP is allowed for localhost loopback development; URL credentials are not allowed`); + } + const apiKeyEnv = value.apiKeyEnv ?? DEFAULT_SEMANTIC_JUDGE_CONFIG.apiKeyEnv; + if (typeof apiKeyEnv !== "string" || !/^[A-Z][A-Z0-9_]{0,127}$/u.test(apiKeyEnv)) { + throw new Error(`Invalid sift-light config at ${path}: semanticJudge.apiKeyEnv must be an uppercase environment variable name`); + } + const model = value.model ?? DEFAULT_SEMANTIC_JUDGE_CONFIG.model; + if (typeof model !== "string" || model.length === 0 || model.length > 128 || /[\r\n\0]/u.test(model)) { + throw new Error(`Invalid sift-light config at ${path}: semanticJudge.model must be bounded single-line text`); + } + return { + enabled, + provider, + endpoint, + apiKeyEnv, + model, + timeoutMs: boundedInteger2(value.timeoutMs, DEFAULT_SEMANTIC_JUDGE_CONFIG.timeoutMs, 1000, 1200000, `config at ${path}: semanticJudge.timeoutMs`), + maxCandidates: boundedInteger2(value.maxCandidates, DEFAULT_SEMANTIC_JUDGE_CONFIG.maxCandidates, 1, 20, `config at ${path}: semanticJudge.maxCandidates`), + maxRetries: boundedInteger2(value.maxRetries, DEFAULT_SEMANTIC_JUDGE_CONFIG.maxRetries, 0, 5, `config at ${path}: semanticJudge.maxRetries`) + }; +} +function parseConfig(value, path) { + if (!isRawSiftLightConfig(value)) { + throw new Error(`Invalid sift-light config at ${path}: expected a JSON object`); + } + const unknown = Object.keys(value).filter((key) => !["locale", "enforceSearch", "vectorSearchEnabled", "semanticJudge"].includes(key)); + if (unknown.length > 0) { + throw new Error(`Invalid sift-light config at ${path}: unsupported configuration fields; only locale, enforceSearch, vectorSearchEnabled and semanticJudge are accepted`); + } + const { locale, enforceSearch, vectorSearchEnabled } = value; + if (locale !== undefined && locale !== "en" && locale !== "zh-CN") { + throw new Error(`Invalid sift-light config at ${path}: locale must be "en" or "zh-CN"`); + } + if (vectorSearchEnabled !== undefined && typeof vectorSearchEnabled !== "boolean") { + throw new Error(`Invalid sift-light config at ${path}: vectorSearchEnabled must be a boolean`); + } + const enforcement = normalizeSearchEnforcement(enforceSearch, `config at ${path}`); + return { + locale: locale ?? DEFAULT_SIFT_LIGHT_CONFIG.locale, + enforceSearch: enforcement, + vectorSearchEnabled: vectorSearchEnabled ?? DEFAULT_SIFT_LIGHT_CONFIG.vectorSearchEnabled ?? false, + semanticJudge: parseSemanticJudge(value.semanticJudge, path) + }; +} +function normalizeSearchEnforcement(value, source) { + if (value === undefined || value === "hard") + return "hard"; + if (value === "prefer") + return "prefer"; + if (value === "off") + return "off"; + throw new Error(`Invalid sift-light ${source}: enforceSearch must be "hard", "prefer", or "off"`); +} +async function readSiftLightConfigFile(path, options = {}) { + try { + const content = await readFile2(path, "utf8"); + return parseConfig(JSON.parse(content), path); + } catch (error) { + if (isMissingFile(error)) { + if (options.missing === "error") { + throw new Error(`sift-light config was not found at ${path}; create it or unset ${SIFT_LIGHT_CONFIG_ENV}`, { cause: error }); + } + return { ...DEFAULT_SIFT_LIGHT_CONFIG }; + } + if (error instanceof SyntaxError) { + throw new Error(`Invalid sift-light config at ${path}: ${error.message}`, { + cause: error + }); + } + throw error; + } +} + // src/semantic-judge-batches.ts var MAX_SEMANTIC_JUDGE_BATCH_CANDIDATES = 8; var MAX_SEMANTIC_JUDGE_REQUEST_BYTES = 64 * 1024; @@ -9872,6 +10026,10 @@ function createSemanticJudgeIntegration(config, environment = process.env, fetch } return { config, runner: createJevRunner(config, key, fetcher) }; } +function createConfiguredSemanticJudgeIntegration(config, environment = process.env) { + const judge = config.semanticJudge ?? DEFAULT_SEMANTIC_JUDGE_CONFIG; + return config.vectorSearchEnabled === true ? createSemanticJudgeIntegration(judge, environment) : createDisabledSemanticJudgeIntegration(judge); +} function baseDetails(config) { return { enabled: config.enabled, @@ -11087,7 +11245,7 @@ import { resolve as resolve24 } from "node:path"; var DISCOVERY_MODE_REQUIRED_ERROR = 'query requires an explicit discovery mode: use mode=files for filename/path discovery or mode=concept for semantic discovery; for example {"mode":"files","query":""}'; // src/format.ts -import { readFile as readFile2 } from "node:fs/promises"; +import { readFile as readFile3 } from "node:fs/promises"; var RESULT_METADATA_RESERVE_BYTES = 1024; var RESULT_METADATA_RESERVE_CHARACTERS = 512; @@ -11221,7 +11379,7 @@ async function loadContextLines(match, expectedRevision, cache, signal) { cache.set(match.absolutePath, changed); return changed; } - const content = await readFile2(match.absolutePath, { encoding: "utf8", signal }); + const content = await readFile3(match.absolutePath, { encoding: "utf8", signal }); const afterRevision = await getSourceRevision(match.absolutePath); if (!afterRevision || !sameSourceRevision(expectedRevision, afterRevision)) { const changed = { status: "changed" }; @@ -12652,6 +12810,7 @@ class SiftLightService { #runRipgrep; #snapshots; #summaryFileLimit; + #vectorSearchEnabled; #capabilities = new LanguageCapabilityCatalog; #evidence; #lifecycle = new AbortController; @@ -12662,12 +12821,16 @@ class SiftLightService { this.#runRipgrep = options.runRipgrep; this.#snapshots = options.snapshots ?? new SnapshotStore; this.#summaryFileLimit = options.summaryFileLimit ?? DEFAULT_SUMMARY_FILE_LIMIT; + this.#vectorSearchEnabled = options.vectorSearchEnabled ?? options.conceptSearch !== undefined; this.#operations = new OperationLifecycle({ deadlineMs: resolveConceptTimeoutMs() }); this.#evidence = new EvidenceService(this.#runRipgrep, this.#snapshots, options.structure, options.conceptSearch, options.semanticJudge); } async search(input, cwd, signal, options = {}) { validateRawSearchInput(input); validateRequestContract(input); + if (!this.#vectorSearchEnabled && (input.mode === "concept" || input.mode === "hybrid")) { + throw new SiftLightError(`${input.mode} search is disabled; set vectorSearchEnabled to true in sift-light.json and restart the host`); + } let request; if (input.mode === "await" || input.mode === "cancel") { request = this.#operationCommand(input, cwd, signal); @@ -13066,160 +13229,11 @@ ${page.body}${rangeNote}${contextNote}${missingSelectionNote} } } -// src/config-reader.ts -import { readFile as readFile3 } from "node:fs/promises"; -var SIFT_LIGHT_CONFIG_ENV = "SIFT_LIGHT_CONFIG"; -var SEMANTIC_JUDGE_API_KEY_ENVS = ["TYPESAFE_API_KEY", "SIFT_LIGHT_JEV_API_KEY"]; -var DEFAULT_SEMANTIC_JUDGE_CONFIG = { - enabled: false, - provider: "jev", - endpoint: "https://api.typesafe.ai/v1/systemone", - apiKeyEnv: SEMANTIC_JUDGE_API_KEY_ENVS[0], - model: "jev-latest", - timeoutMs: 120000, - maxCandidates: 20, - maxRetries: 2 -}; -var DEFAULT_SIFT_LIGHT_CONFIG = { - locale: "en", - enforceSearch: "hard", - semanticJudge: DEFAULT_SEMANTIC_JUDGE_CONFIG -}; -function hasErrorCode(error, codes) { - return error instanceof Error && "code" in error && codes.includes(String(error.code)); -} -function isMissingFile(error) { - return hasErrorCode(error, ["ENOENT"]); -} -function isRawSiftLightConfig(value) { - return typeof value === "object" && value !== null && !Array.isArray(value); -} -function isRawSemanticJudgeConfig(value) { - return typeof value === "object" && value !== null && !Array.isArray(value); -} -function boundedInteger2(value, fallback, minimum, maximum, field) { - const candidate = value ?? fallback; - if (typeof candidate !== "number" || !Number.isSafeInteger(candidate) || candidate < minimum || candidate > maximum) { - throw new Error(`Invalid sift-light ${field}: expected an integer from ${String(minimum)} through ${String(maximum)}`); - } - return candidate; -} -function parseSemanticJudge(value, path) { - if (value === undefined) - return { ...DEFAULT_SEMANTIC_JUDGE_CONFIG }; - if (!isRawSemanticJudgeConfig(value)) { - throw new Error(`Invalid sift-light config at ${path}: semanticJudge must be an object`); - } - const unknown = Object.keys(value).filter((key) => ![ - "enabled", - "provider", - "endpoint", - "apiKeyEnv", - "model", - "timeoutMs", - "maxCandidates", - "maxRetries" - ].includes(key)); - if (unknown.length > 0) { - throw new Error(`Invalid sift-light config at ${path}: unsupported semanticJudge fields; accepted fields are enabled, provider, endpoint, apiKeyEnv, model, timeoutMs, maxCandidates and maxRetries`); - } - const enabled = value.enabled ?? DEFAULT_SEMANTIC_JUDGE_CONFIG.enabled; - if (typeof enabled !== "boolean") { - throw new Error(`Invalid sift-light config at ${path}: semanticJudge.enabled must be a boolean`); - } - const provider = value.provider ?? DEFAULT_SEMANTIC_JUDGE_CONFIG.provider; - if (provider !== "jev") { - throw new Error(`Invalid sift-light config at ${path}: semanticJudge.provider must be "jev"`); - } - const endpoint = value.endpoint ?? DEFAULT_SEMANTIC_JUDGE_CONFIG.endpoint; - if (typeof endpoint !== "string" || endpoint.length === 0 || endpoint.length > 2048) { - throw new Error(`Invalid sift-light config at ${path}: semanticJudge.endpoint must be a nonempty URL`); - } - let parsedEndpoint; - try { - parsedEndpoint = new URL(endpoint); - } catch (error) { - throw new Error(`Invalid sift-light config at ${path}: semanticJudge.endpoint must be a URL`, { - cause: error - }); - } - const loopbackHosts = new Set(["localhost", "127.0.0.1", "[::1]"]); - const secureEndpoint = parsedEndpoint.protocol === "https:" || parsedEndpoint.protocol === "http:" && loopbackHosts.has(parsedEndpoint.hostname); - if (!secureEndpoint || parsedEndpoint.username || parsedEndpoint.password) { - throw new Error(`Invalid sift-light config at ${path}: semanticJudge.endpoint must use HTTPS, except that HTTP is allowed for localhost loopback development; URL credentials are not allowed`); - } - const apiKeyEnv = value.apiKeyEnv ?? DEFAULT_SEMANTIC_JUDGE_CONFIG.apiKeyEnv; - if (typeof apiKeyEnv !== "string" || !/^[A-Z][A-Z0-9_]{0,127}$/u.test(apiKeyEnv)) { - throw new Error(`Invalid sift-light config at ${path}: semanticJudge.apiKeyEnv must be an uppercase environment variable name`); - } - const model = value.model ?? DEFAULT_SEMANTIC_JUDGE_CONFIG.model; - if (typeof model !== "string" || model.length === 0 || model.length > 128 || /[\r\n\0]/u.test(model)) { - throw new Error(`Invalid sift-light config at ${path}: semanticJudge.model must be bounded single-line text`); - } - return { - enabled, - provider, - endpoint, - apiKeyEnv, - model, - timeoutMs: boundedInteger2(value.timeoutMs, DEFAULT_SEMANTIC_JUDGE_CONFIG.timeoutMs, 1000, 1200000, `config at ${path}: semanticJudge.timeoutMs`), - maxCandidates: boundedInteger2(value.maxCandidates, DEFAULT_SEMANTIC_JUDGE_CONFIG.maxCandidates, 1, 20, `config at ${path}: semanticJudge.maxCandidates`), - maxRetries: boundedInteger2(value.maxRetries, DEFAULT_SEMANTIC_JUDGE_CONFIG.maxRetries, 0, 5, `config at ${path}: semanticJudge.maxRetries`) - }; -} -function parseConfig(value, path) { - if (!isRawSiftLightConfig(value)) { - throw new Error(`Invalid sift-light config at ${path}: expected a JSON object`); - } - const unknown = Object.keys(value).filter((key) => !["locale", "enforceSearch", "semanticJudge"].includes(key)); - if (unknown.length > 0) { - throw new Error(`Invalid sift-light config at ${path}: unsupported configuration fields; only locale, enforceSearch and semanticJudge are accepted`); - } - const { locale, enforceSearch } = value; - if (locale !== undefined && locale !== "en" && locale !== "zh-CN") { - throw new Error(`Invalid sift-light config at ${path}: locale must be "en" or "zh-CN"`); - } - const enforcement = normalizeSearchEnforcement(enforceSearch, `config at ${path}`); - return { - locale: locale ?? DEFAULT_SIFT_LIGHT_CONFIG.locale, - enforceSearch: enforcement, - semanticJudge: parseSemanticJudge(value.semanticJudge, path) - }; -} -function normalizeSearchEnforcement(value, source) { - if (value === undefined || value === "hard") - return "hard"; - if (value === "prefer") - return "prefer"; - if (value === "off") - return "off"; - throw new Error(`Invalid sift-light ${source}: enforceSearch must be "hard", "prefer", or "off"`); -} -async function readSiftLightConfigFile(path, options = {}) { - try { - const content = await readFile3(path, "utf8"); - return parseConfig(JSON.parse(content), path); - } catch (error) { - if (isMissingFile(error)) { - if (options.missing === "error") { - throw new Error(`sift-light config was not found at ${path}; create it or unset ${SIFT_LIGHT_CONFIG_ENV}`, { cause: error }); - } - return { ...DEFAULT_SIFT_LIGHT_CONFIG }; - } - if (error instanceof SyntaxError) { - throw new Error(`Invalid sift-light config at ${path}: ${error.message}`, { - cause: error - }); - } - throw error; - } -} - // src/prompt-guidelines.ts var SOURCE_OUTPUT_GUIDANCE = "Auto/summary text may include bounded source excerpts; ordinary matches text is metadata-only. Inspect may return bounded source windows covering an entire small file. Analysis text may include semantic passages; structured details may retain excerpts, names and signatures. Follow output limits, coverage and continuations."; function siftLightPromptGuidelines(structuredOutput = true) { return [ - `Use sift-light for read-only content search. ${SOURCE_OUTPUT_GUIDANCE} Omit mode and limit for automatic detail/summary selection; use mode="matches" for ordinary match metadata.`, + `Use sift-light for read-only content search. ${SOURCE_OUTPUT_GUIDANCE} For routine development searches, start with fast exact content, filename or applicable structural modes when the request has a usable name, symbol, error text or other literal clue. Omitted mode is ordinary exact search and never loads the local embedding model. Vector search is disabled by default; concept/hybrid require vectorSearchEnabled:true in sift-light.json and an installed model. They can take tens of seconds on an uncached scope, so select them only when semantic recall is needed. Omit mode and limit for automatic detail/summary selection; use mode="matches" for ordinary match metadata.`, `An omitted path searches the project cwd. Use scope:"strict" for a question restricted to one path; otherwise, if an explicit subpath has zero matches, ordinary and content-analysis searches retry from cwd and return project-wide counts with an expansion notice. Explicit absolute paths and .. traversal can search outside cwd, except protected external system areas and .git internals. Git changes mode remains cwd-scoped.`, `Search output includes counts, categories, ranked paths, coverage and continuation metadata. Source excerpts may contain the searched text. Use mode="inspect" or the host read capability when exact source is required for an edit or verification.`, `Use file and directory distributions to choose evidence. Reuse the visible cursor with path or paths for match metadata; mode="summary" pages the remaining file statistics. Match counts are not relevance scores.`, @@ -13228,7 +13242,7 @@ function siftLightPromptGuidelines(structuredOutput = true) { `Use anyOf:["term1","term2"] when every exact occurrence of 2-64 literals is needed in one version-bound result. It is case-sensitive, reports anonymized condition counts, and runs requests above eight terms as bounded parallel chunks. Large condition inventories have separate continuation pages; copy those requests to retrieve the complete counts.`, `For a changed-code question, add changes:{base:"HEAD",scope:"lines",side:"new"}; omit target for the working tree, use side:"old" for deleted-side statistics. Copy returned continuation requests to preserve source versions.`, `Use mode:"capabilities" when the language or requested operation is unclear to get a compact lazy inventory. Use mode:"outline" with a concrete source file path for symbol counts and locations, mode:"imports" for static relationships, and mode:"tests" for related-test candidates. Their text pages summarize metadata; structured details can retain source evidence.`, - `Use mode:"files" plus query for unknown filenames and fuzzy paths. Multi-word filename queries require each word literally in the path; business concepts belong in hybrid/concept. Use wholeWord:true for a single-pattern whole-word search. exclude contains file globs, not content negation.`, + `Use mode:"files" plus query for unknown filenames and fuzzy paths. Multi-word filename queries require each word literally in the path; use hybrid/concept only for business concepts that cannot be located by a literal clue. Use wholeWord:true for a single-pattern whole-word search. exclude contains file globs, not content negation.`, `Use mode:"structure" only for JS/TS/TSX/Go, with a required nonempty ast-grep pattern such as "compare($X, $X)" or "send()" for code shapes across whitespace; the text page reports structural counts and locations; structured details can retain matched source evidence.`, `Use mode:"concept" plus a natural-language query when names are unknown. It runs a pinned local multilingual model only after explicit installation; no search downloads weights or sends code to a remote model. Results expose candidate counts, score statistics, paths and the bounded ranked passage behind each candidate. Similarity scores identify candidates, not correctness.`, `Use mode:"hybrid" plus query when wording may differ from the source. It reports exact and semantic counts separately, removes overlap, and retains one pageable evidence snapshot. conceptLimit changes only the semantic candidate count; retained candidates keep their bounded source passages.`, @@ -13239,9 +13253,9 @@ function siftLightPromptGuidelines(structuredOutput = true) { } function siftLightModelGuidelines() { return [ - `Search with pattern and optional path. ${SOURCE_OUTPUT_GUIDANCE} Omit mode/limit for automatic detail/summary selection. Compact model analysis pages may defer source excerpts to inspect.`, - `Use ranked paths and condition counts to choose evidence. Cursor continuation pages the retained snapshot; summary file selection returns ordinary match metadata. Use mode="inspect" when an edit requires exact source.`, - `Modes: capabilities for language prerequisites; files+query for filenames; anyOf/allOf for literal condition statistics; outline/imports/tests for structural and relationship summaries; structure for AST match statistics; concept/hybrid for semantic candidate statistics. Similarity and static links are not proof.`, + `Search with pattern and optional path. ${SOURCE_OUTPUT_GUIDANCE} Default search is exact and model-free. concept/hybrid require vectorSearchEnabled:true in sift-light.json plus an installed model; uncached runs may take tens of seconds. Omit mode/limit for auto pages.`, + `Use ranked paths and counts; Cursor continuation pages retained results. Use mode="inspect" for exact source before editing.`, + `Modes: files+query for filenames; anyOf/allOf for literals; outline/imports/tests for static code; structure for AST; concept/hybrid for semantic candidates. Similarity is not proof.`, `On rejection keep the strongest applicable mode and apply the stated repair once. Do not repeat the rejected request or include its error. Only explicit capability-unavailable permits a visibly partial alternative.` ]; } @@ -13264,12 +13278,12 @@ function stringEnum(values, options) { ...options?.description ? { description: options.description } : {} }); } -var SIFT_LIGHT_DESCRIPTION = `Search and navigate code with bounded, verifiable evidence. Ordinary pattern searches use auto detail/summary; pattern is regex by default and literal=true matches source text exactly. A path selects an existing exact file or root; use mode=files with query to discover an unknown name. scope=strict prevents zero-result path expansion and wholeWord requires word boundaries. mode=capabilities returns a compact names-only project language inventory and the modes available for each detected language; capability providers are loaded only when the requested analysis runs. It never starts a parser, compiler, model or language server. mode=concept accepts a natural-language query, path and source filters; mode=hybrid uses one natural-language query for exact and local concept evidence, ranks exact evidence first, and retains a bounded semantic supplement. An explicitly enabled semantic judge may classify hybrid candidates, but it is disabled by default and never turns classification into a runtime proof. Slow concept/hybrid requests return status=waiting or running with operationId, progress, and an exact nextRequest using mode=await; copy that request unchanged to continue the same computation. Await expiry never downgrades evidence to literal-only or partial, and final results remain stable for the operation retention window. mode=cancel explicitly stops one operation. allOf and anyOf are explicit literal variants and cannot be mixed with pattern/literal; limit and context are output intent and are never silently dropped. modifiedAfter/modifiedBefore filter worktree files by inclusive/exclusive modification-time bounds in Unix milliseconds. structure requires a nonempty AST pattern and JS/TS/TSX/Go sources; lang is not a field. Outline uses a concrete source file path (not a directory) or retained cursor+matchIndex and follows declared syntax capabilities. imports/tests return bounded static module and related-test candidates without proving runtime execution. validate checks saved source evidence against its recorded origin. Partial coverage stays explicit. ${REQUEST_USAGE_GUIDANCE}`; -var SIFT_LIGHT_MODEL_DESCRIPTION = `Bounded local evidence search. ${MODEL_USAGE_GUIDANCE}. Copy cursors; analysis is evidence, not proof.`; +var SIFT_LIGHT_DESCRIPTION = `Search and navigate code with bounded, verifiable evidence. Routine searches should use exact content, filenames or applicable structural modes first. Omitted mode is ordinary exact search and does not load the embedding model; concept/hybrid require vectorSearchEnabled:true in sift-light.json plus an installed model and may take tens of seconds on an uncached scope. Ordinary pattern searches use auto detail/summary; pattern is regex by default and literal=true matches source text exactly. A path selects an existing exact file or root; use mode=files with query to discover an unknown name. scope=strict prevents zero-result path expansion and wholeWord requires word boundaries. mode=capabilities returns a compact names-only project language inventory and the modes available for each detected language; capability providers are loaded only when the requested analysis runs. It never starts a parser, compiler, model or language server. mode=concept accepts a natural-language query, path and source filters; mode=hybrid uses one natural-language query for exact and local concept evidence, ranks exact evidence first, and retains a bounded semantic supplement. An explicitly enabled semantic judge may classify hybrid candidates, but it is disabled by default and never turns classification into a runtime proof. Slow concept/hybrid requests return status=waiting or running with operationId, progress, and an exact nextRequest using mode=await; copy that request unchanged to continue the same computation. Await expiry never downgrades evidence to literal-only or partial, and final results remain stable for the operation retention window. mode=cancel explicitly stops one operation. allOf and anyOf are explicit literal variants and cannot be mixed with pattern/literal; limit and context are output intent and are never silently dropped. modifiedAfter/modifiedBefore filter worktree files by inclusive/exclusive modification-time bounds in Unix milliseconds. structure requires a nonempty AST pattern and JS/TS/TSX/Go sources; lang is not a field. Outline uses a concrete source file path (not a directory) or retained cursor+matchIndex and follows declared syntax capabilities. imports/tests return bounded static module and related-test candidates without proving runtime execution. validate checks saved source evidence against its recorded origin. Partial coverage stays explicit. ${REQUEST_USAGE_GUIDANCE}`; +var SIFT_LIGHT_MODEL_DESCRIPTION = `Bounded local evidence search. Default exact search is model-free; concept/hybrid require vectorSearchEnabled:true in sift-light.json and a model. ${MODEL_USAGE_GUIDANCE}. Copy cursors; analysis is evidence, not proof.`; var siftLightSchema = Type.Object({ query: Type.Optional(Type.String({ maxLength: 256, - description: `${fieldGuidance("query")}. Hybrid uses the same query as exact literal text and as the local concept query. Discovery modes preserve their requested path. Concept and hybrid require an explicitly installed local model. A semantic judge is optional and remains disabled unless the active configuration explicitly enables it.` + description: `${fieldGuidance("query")}. Hybrid uses the same query as exact literal text and as the local concept query. Discovery modes preserve their requested path. Concept and hybrid require vectorSearchEnabled:true in sift-light.json and an explicitly installed local model. A semantic judge is optional and remains disabled unless the active configuration explicitly enables it.` })), scope: Type.Optional(stringEnum(["strict", "expand"], { description: `${fieldGuidance("scope")}; expand (default) retries ordinary content search from project cwd. Applies to ordinary, multi-term and role searches.` @@ -13472,11 +13486,12 @@ function siftLightTool(outputMode) { tool.outputSchema = SIFT_LIGHT_OUTPUT_SCHEMA; return tool; } -function createDefaultSiftLightMcpService(semanticJudge) { +function createDefaultSiftLightMcpService(semanticJudge, vectorSearchEnabled = false) { const resolvedSemanticJudge = semanticJudge ?? createDisabledSemanticJudgeIntegration(DEFAULT_SEMANTIC_JUDGE_CONFIG); return new SiftLightService({ runRipgrep: createRipgrepRunner(), structure: createCtagsStructureProvider(), + vectorSearchEnabled, semanticJudge: resolvedSemanticJudge }); } @@ -13899,12 +13914,18 @@ function configuredPath(environment) { const value = environment[SIFT_LIGHT_CONFIG_ENV]?.trim(); return value || undefined; } -async function createMcpSemanticJudgeIntegration(environment = process.env) { +async function createMcpSearchFeatures(environment = process.env) { const path = configuredPath(environment); if (!path) - return createDisabledSemanticJudgeIntegration(DEFAULT_SEMANTIC_JUDGE_CONFIG); + return { + semanticJudge: createDisabledSemanticJudgeIntegration(DEFAULT_SEMANTIC_JUDGE_CONFIG), + vectorSearchEnabled: DEFAULT_SIFT_LIGHT_CONFIG.vectorSearchEnabled === true + }; const config = await readSiftLightConfigFile(path, { missing: "error" }); - return createSemanticJudgeIntegration(config.semanticJudge ?? DEFAULT_SEMANTIC_JUDGE_CONFIG, environment); + return { + semanticJudge: createConfiguredSemanticJudgeIntegration(config, environment), + vectorSearchEnabled: config.vectorSearchEnabled === true + }; } function mcpSemanticJudgeConfigSource(environment = process.env) { return configuredPath(environment) ? "explicit-config" : "not-configured"; @@ -14018,20 +14039,17 @@ function environmentInteger(name, fallback, minimum, maximum) { function allowedOrigins() { return (process.env.SIFT_LIGHT_MCP_ALLOWED_ORIGINS ?? "").split(",").map((origin) => origin.trim()).filter((origin) => origin.length > 0); } -async function configuredSemanticJudge() { - return createMcpSemanticJudgeIntegration(); -} function logSemanticJudgeStatus(integration) { const status = integration.config.enabled ? "enabled" : "disabled"; process.stderr.write(`sift-light MCP semantic judge: ${status}; source=${mcpSemanticJudgeConfigSource()} `); } -async function runHttpServer(outputMode, semanticJudge) { +async function runHttpServer(outputMode, semanticJudge, vectorSearchEnabled) { const running = await startSiftLightMcpServer({ cwd: process.env.SIFT_LIGHT_MCP_CWD ?? process.cwd(), host: process.env.SIFT_LIGHT_MCP_HOST ?? DEFAULT_MCP_HOST, port: environmentInteger("SIFT_LIGHT_MCP_PORT", DEFAULT_MCP_PORT, 0, 65535), - createService: () => createDefaultSiftLightMcpService(semanticJudge), + createService: () => createDefaultSiftLightMcpService(semanticJudge, vectorSearchEnabled), maxSessions: environmentInteger("SIFT_LIGHT_MCP_MAX_SESSIONS", DEFAULT_MCP_MAX_SESSIONS, 1, Number.MAX_SAFE_INTEGER), sessionIdleTimeoutMs: environmentInteger("SIFT_LIGHT_MCP_SESSION_IDLE_MS", DEFAULT_MCP_SESSION_IDLE_TIMEOUT_MS, 1, Number.MAX_SAFE_INTEGER), allowedOrigins: allowedOrigins(), @@ -14065,11 +14083,11 @@ async function runHttpServer(outputMode, semanticJudge) { process.once("SIGINT", shutdown); process.once("SIGTERM", shutdown); } -async function runStdioServer(outputMode, semanticJudge) { +async function runStdioServer(outputMode, semanticJudge, vectorSearchEnabled) { const running = await startSiftLightMcpStdioServer({ cwd: process.env.SIFT_LIGHT_MCP_CWD ?? process.env.CLAUDE_PROJECT_DIR ?? process.cwd(), outputMode, - createService: () => createDefaultSiftLightMcpService(semanticJudge) + createService: () => createDefaultSiftLightMcpService(semanticJudge, vectorSearchEnabled) }); process.stderr.write(`sift-light MCP serving one local client over stdio `); @@ -14098,13 +14116,15 @@ async function main() { return; } const outputMode = parseSiftLightMcpOutputMode(process.env.SIFT_LIGHT_MCP_OUTPUT_MODE); - const semanticJudge = await configuredSemanticJudge(); + const { semanticJudge, vectorSearchEnabled } = await createMcpSearchFeatures(); logSemanticJudgeStatus(semanticJudge); + process.stderr.write(`sift-light MCP vector search: ${vectorSearchEnabled ? "enabled" : "disabled"} +`); if (transport === "stdio") { - await runStdioServer(outputMode, semanticJudge); + await runStdioServer(outputMode, semanticJudge, vectorSearchEnabled); return; } - await runHttpServer(outputMode, semanticJudge); + await runHttpServer(outputMode, semanticJudge, vectorSearchEnabled); } try { await main(); diff --git a/src/mcp-server.ts b/src/mcp-server.ts index 8839f899..02139129 100644 --- a/src/mcp-server.ts +++ b/src/mcp-server.ts @@ -8,10 +8,7 @@ import { createDefaultSiftLightMcpService, startSiftLightMcpServer, } from "./mcp.js"; -import { - createMcpSemanticJudgeIntegration, - mcpSemanticJudgeConfigSource, -} from "./mcp-semantic-judge.js"; +import { createMcpSearchFeatures, mcpSemanticJudgeConfigSource } from "./mcp-semantic-judge.js"; import type { SemanticJudgeIntegration } from "./semantic-judge.js"; import { parseSiftLightMcpTransport, SIFT_LIGHT_MCP_USAGE } from "./mcp-cli.js"; import { parseSiftLightMcpOutputMode, type SiftLightMcpOutputMode } from "./mcp-output.js"; @@ -41,10 +38,6 @@ function allowedOrigins(): string[] { .filter((origin) => origin.length > 0); } -async function configuredSemanticJudge(): Promise { - return createMcpSemanticJudgeIntegration(); -} - function logSemanticJudgeStatus(integration: SemanticJudgeIntegration): void { const status = integration.config.enabled ? "enabled" : "disabled"; process.stderr.write( @@ -55,12 +48,13 @@ function logSemanticJudgeStatus(integration: SemanticJudgeIntegration): void { async function runHttpServer( outputMode: SiftLightMcpOutputMode, semanticJudge: SemanticJudgeIntegration | undefined, + vectorSearchEnabled: boolean, ): Promise { const running = await startSiftLightMcpServer({ cwd: process.env.SIFT_LIGHT_MCP_CWD ?? process.cwd(), host: process.env.SIFT_LIGHT_MCP_HOST ?? DEFAULT_MCP_HOST, port: environmentInteger("SIFT_LIGHT_MCP_PORT", DEFAULT_MCP_PORT, 0, 65_535), - createService: () => createDefaultSiftLightMcpService(semanticJudge), + createService: () => createDefaultSiftLightMcpService(semanticJudge, vectorSearchEnabled), maxSessions: environmentInteger( "SIFT_LIGHT_MCP_MAX_SESSIONS", DEFAULT_MCP_MAX_SESSIONS, @@ -108,11 +102,12 @@ async function runHttpServer( async function runStdioServer( outputMode: SiftLightMcpOutputMode, semanticJudge: SemanticJudgeIntegration | undefined, + vectorSearchEnabled: boolean, ): Promise { const running = await startSiftLightMcpStdioServer({ cwd: process.env.SIFT_LIGHT_MCP_CWD ?? process.env.CLAUDE_PROJECT_DIR ?? process.cwd(), outputMode, - createService: () => createDefaultSiftLightMcpService(semanticJudge), + createService: () => createDefaultSiftLightMcpService(semanticJudge, vectorSearchEnabled), }); process.stderr.write("sift-light MCP serving one local client over stdio\n"); process.stderr.write(`sift-light MCP working directory: ${running.cwd}\n`); @@ -140,13 +135,16 @@ async function main(): Promise { return; } const outputMode = parseSiftLightMcpOutputMode(process.env.SIFT_LIGHT_MCP_OUTPUT_MODE); - const semanticJudge = await configuredSemanticJudge(); + const { semanticJudge, vectorSearchEnabled } = await createMcpSearchFeatures(); logSemanticJudgeStatus(semanticJudge); + process.stderr.write( + `sift-light MCP vector search: ${vectorSearchEnabled ? "enabled" : "disabled"}\n`, + ); if (transport === "stdio") { - await runStdioServer(outputMode, semanticJudge); + await runStdioServer(outputMode, semanticJudge, vectorSearchEnabled); return; } - await runHttpServer(outputMode, semanticJudge); + await runHttpServer(outputMode, semanticJudge, vectorSearchEnabled); } try { diff --git a/src/mcp.ts b/src/mcp.ts index 12d7696d..733fe0ac 100644 --- a/src/mcp.ts +++ b/src/mcp.ts @@ -93,12 +93,14 @@ export interface SiftLightMcpService { export function createDefaultSiftLightMcpService( semanticJudge?: SemanticJudgeIntegration, + vectorSearchEnabled = false, ): SiftLightMcpService { const resolvedSemanticJudge = semanticJudge ?? createDisabledSemanticJudgeIntegration(DEFAULT_SEMANTIC_JUDGE_CONFIG); return new SiftLightService({ runRipgrep: createRipgrepRunner(), structure: createCtagsStructureProvider(), + vectorSearchEnabled, semanticJudge: resolvedSemanticJudge, }); } diff --git a/src/omp-index.ts b/src/omp-index.ts index a89feaac..9b5d406a 100644 --- a/src/omp-index.ts +++ b/src/omp-index.ts @@ -1,7 +1,6 @@ import { homedir } from "node:os"; import { join } from "node:path"; import { - DEFAULT_SEMANTIC_JUDGE_CONFIG, normalizeSearchEnforcement, readSiftLightConfigFile, resolveSiftLightConfigPath, @@ -27,7 +26,7 @@ import { } from "./search-policy.js"; import { siftLightSchema } from "./tool-schema.js"; import { modelErrorText } from "./model-error.js"; -import { createSemanticJudgeIntegration } from "./semantic-judge.js"; +import { createConfiguredSemanticJudgeIntegration } from "./semantic-judge.js"; const SIFT_LIGHT_LABEL = "sift-light"; const OMP_REPLACED_SEARCH_TOOLS = new Set(["grep", "glob"]); @@ -188,13 +187,12 @@ export async function registerOmpSiftLightExtension( (await readSiftLightConfigFile(resolveSiftLightConfigPath(ompAgentDir()), { missing: process.env[SIFT_LIGHT_CONFIG_ENV]?.trim() ? "error" : "defaults", })); - const semanticJudge = createSemanticJudgeIntegration( - resolvedConfig.semanticJudge ?? DEFAULT_SEMANTIC_JUDGE_CONFIG, - ); + const semanticJudge = createConfiguredSemanticJudgeIntegration(resolvedConfig); const runtime = new SiftLightRuntime( new SiftLightService({ runRipgrep: createRipgrepRunner(), structure: createCtagsStructureProvider(), + vectorSearchEnabled: resolvedConfig.vectorSearchEnabled === true, semanticJudge, }), ); @@ -218,7 +216,7 @@ export async function registerOmpSiftLightExtension( name: SIFT_LIGHT_LABEL, label: SIFT_LIGHT_LABEL, description: - "Search and navigate code with bounded, verifiable evidence. Use pattern for content or mode=files with query for filenames.", + "Search code with bounded evidence. Routine searches use pattern for fast exact content or mode=files with query for filenames; concept/hybrid require vectorSearchEnabled:true in sift-light.json.", approval: "read", promptSnippet: "Search file contents without flooding context", promptGuidelines: siftLightPromptGuidelines(), diff --git a/src/prompt-guidelines.ts b/src/prompt-guidelines.ts index ed578bbc..44dd49ba 100644 --- a/src/prompt-guidelines.ts +++ b/src/prompt-guidelines.ts @@ -5,7 +5,7 @@ const SOURCE_OUTPUT_GUIDANCE = export function siftLightPromptGuidelines(structuredOutput = true): string[] { return [ - `Use sift-light for read-only content search. ${SOURCE_OUTPUT_GUIDANCE} Omit mode and limit for automatic detail/summary selection; use mode="matches" for ordinary match metadata.`, + `Use sift-light for read-only content search. ${SOURCE_OUTPUT_GUIDANCE} For routine development searches, start with fast exact content, filename or applicable structural modes when the request has a usable name, symbol, error text or other literal clue. Omitted mode is ordinary exact search and never loads the local embedding model. Vector search is disabled by default; concept/hybrid require vectorSearchEnabled:true in sift-light.json and an installed model. They can take tens of seconds on an uncached scope, so select them only when semantic recall is needed. Omit mode and limit for automatic detail/summary selection; use mode="matches" for ordinary match metadata.`, `An omitted path searches the project cwd. Use scope:"strict" for a question restricted to one path; otherwise, if an explicit subpath has zero matches, ordinary and content-analysis searches retry from cwd and return project-wide counts with an expansion notice. Explicit absolute paths and .. traversal can search outside cwd, except protected external system areas and .git internals. Git changes mode remains cwd-scoped.`, `Search output includes counts, categories, ranked paths, coverage and continuation metadata. Source excerpts may contain the searched text. Use mode="inspect" or the host read capability when exact source is required for an edit or verification.`, `Use file and directory distributions to choose evidence. Reuse the visible cursor with path or paths for match metadata; mode="summary" pages the remaining file statistics. Match counts are not relevance scores.`, @@ -14,7 +14,7 @@ export function siftLightPromptGuidelines(structuredOutput = true): string[] { `Use anyOf:["term1","term2"] when every exact occurrence of 2-64 literals is needed in one version-bound result. It is case-sensitive, reports anonymized condition counts, and runs requests above eight terms as bounded parallel chunks. Large condition inventories have separate continuation pages; copy those requests to retrieve the complete counts.`, `For a changed-code question, add changes:{base:"HEAD",scope:"lines",side:"new"}; omit target for the working tree, use side:"old" for deleted-side statistics. Copy returned continuation requests to preserve source versions.`, `Use mode:"capabilities" when the language or requested operation is unclear to get a compact lazy inventory. Use mode:"outline" with a concrete source file path for symbol counts and locations, mode:"imports" for static relationships, and mode:"tests" for related-test candidates. Their text pages summarize metadata; structured details can retain source evidence.`, - `Use mode:"files" plus query for unknown filenames and fuzzy paths. Multi-word filename queries require each word literally in the path; business concepts belong in hybrid/concept. Use wholeWord:true for a single-pattern whole-word search. exclude contains file globs, not content negation.`, + `Use mode:"files" plus query for unknown filenames and fuzzy paths. Multi-word filename queries require each word literally in the path; use hybrid/concept only for business concepts that cannot be located by a literal clue. Use wholeWord:true for a single-pattern whole-word search. exclude contains file globs, not content negation.`, `Use mode:"structure" only for JS/TS/TSX/Go, with a required nonempty ast-grep pattern such as "compare($X, $X)" or "send()" for code shapes across whitespace; the text page reports structural counts and locations; structured details can retain matched source evidence.`, `Use mode:"concept" plus a natural-language query when names are unknown. It runs a pinned local multilingual model only after explicit installation; no search downloads weights or sends code to a remote model. Results expose candidate counts, score statistics, paths and the bounded ranked passage behind each candidate. Similarity scores identify candidates, not correctness.`, `Use mode:"hybrid" plus query when wording may differ from the source. It reports exact and semantic counts separately, removes overlap, and retains one pageable evidence snapshot. conceptLimit changes only the semantic candidate count; retained candidates keep their bounded source passages.`, @@ -28,9 +28,9 @@ export function siftLightPromptGuidelines(structuredOutput = true): string[] { function siftLightModelGuidelines(): string[] { return [ - `Search with pattern and optional path. ${SOURCE_OUTPUT_GUIDANCE} Omit mode/limit for automatic detail/summary selection. Compact model analysis pages may defer source excerpts to inspect.`, - `Use ranked paths and condition counts to choose evidence. Cursor continuation pages the retained snapshot; summary file selection returns ordinary match metadata. Use mode="inspect" when an edit requires exact source.`, - `Modes: capabilities for language prerequisites; files+query for filenames; anyOf/allOf for literal condition statistics; outline/imports/tests for structural and relationship summaries; structure for AST match statistics; concept/hybrid for semantic candidate statistics. Similarity and static links are not proof.`, + `Search with pattern and optional path. ${SOURCE_OUTPUT_GUIDANCE} Default search is exact and model-free. concept/hybrid require vectorSearchEnabled:true in sift-light.json plus an installed model; uncached runs may take tens of seconds. Omit mode/limit for auto pages.`, + `Use ranked paths and counts; Cursor continuation pages retained results. Use mode="inspect" for exact source before editing.`, + `Modes: files+query for filenames; anyOf/allOf for literals; outline/imports/tests for static code; structure for AST; concept/hybrid for semantic candidates. Similarity is not proof.`, `On rejection keep the strongest applicable mode and apply the stated repair once. Do not repeat the rejected request or include its error. Only explicit capability-unavailable permits a visibly partial alternative.`, ]; } diff --git a/src/semantic-judge.ts b/src/semantic-judge.ts index b3b7bb75..f94ffb6d 100644 --- a/src/semantic-judge.ts +++ b/src/semantic-judge.ts @@ -1,4 +1,8 @@ -import type { SemanticJudgeConfig } from "./config-reader.js"; +import { + DEFAULT_SEMANTIC_JUDGE_CONFIG, + type SemanticJudgeConfig, + type SiftLightConfig, +} from "./config-reader.js"; import { SEMANTIC_JUDGE_CLASSIFICATIONS, SEMANTIC_JUDGE_NON_PROOF_CLAIM, @@ -373,6 +377,17 @@ export function createSemanticJudgeIntegration( } return { config, runner: createJevRunner(config, key, fetcher) }; } + +export function createConfiguredSemanticJudgeIntegration( + config: SiftLightConfig, + environment: NodeJS.ProcessEnv = process.env, +): SemanticJudgeIntegration { + const judge = config.semanticJudge ?? DEFAULT_SEMANTIC_JUDGE_CONFIG; + return config.vectorSearchEnabled === true + ? createSemanticJudgeIntegration(judge, environment) + : createDisabledSemanticJudgeIntegration(judge); +} + function baseDetails(config: SemanticJudgeConfig): SemanticJudgeDetails { return { enabled: config.enabled, diff --git a/src/service.ts b/src/service.ts index 17d1560d..4510bb2c 100644 --- a/src/service.ts +++ b/src/service.ts @@ -87,6 +87,7 @@ export interface SiftLightServiceOptions { snapshots?: SnapshotStore; summaryFileLimit?: number; structure?: CodeStructureProvider; + vectorSearchEnabled?: boolean; conceptSearch?: ConceptSearchRunner; semanticJudge?: SemanticJudgeIntegration; } @@ -367,6 +368,7 @@ export class SiftLightService { readonly #runRipgrep: RipgrepRunner; readonly #snapshots: SnapshotStore; readonly #summaryFileLimit: number; + readonly #vectorSearchEnabled: boolean; readonly #capabilities = new LanguageCapabilityCatalog(); readonly #evidence: EvidenceService; #lifecycle = new AbortController(); @@ -378,6 +380,7 @@ export class SiftLightService { this.#runRipgrep = options.runRipgrep; this.#snapshots = options.snapshots ?? new SnapshotStore(); this.#summaryFileLimit = options.summaryFileLimit ?? DEFAULT_SUMMARY_FILE_LIMIT; + this.#vectorSearchEnabled = options.vectorSearchEnabled ?? options.conceptSearch !== undefined; this.#operations = new OperationLifecycle({ deadlineMs: resolveConceptTimeoutMs() }); this.#evidence = new EvidenceService( this.#runRipgrep, @@ -396,6 +399,11 @@ export class SiftLightService { ): Promise { validateRawSearchInput(input); validateRequestContract(input); + if (!this.#vectorSearchEnabled && (input.mode === "concept" || input.mode === "hybrid")) { + throw new SiftLightError( + `${input.mode} search is disabled; set vectorSearchEnabled to true in sift-light.json and restart the host`, + ); + } let request: Promise; if (input.mode === "await" || input.mode === "cancel") { request = this.#operationCommand(input, cwd, signal); diff --git a/src/tool-schema.ts b/src/tool-schema.ts index da5a01d2..58297e28 100644 --- a/src/tool-schema.ts +++ b/src/tool-schema.ts @@ -37,15 +37,15 @@ function stringEnum( }); } -export const SIFT_LIGHT_DESCRIPTION = `Search and navigate code with bounded, verifiable evidence. Ordinary pattern searches use auto detail/summary; pattern is regex by default and literal=true matches source text exactly. A path selects an existing exact file or root; use mode=files with query to discover an unknown name. scope=strict prevents zero-result path expansion and wholeWord requires word boundaries. mode=capabilities returns a compact names-only project language inventory and the modes available for each detected language; capability providers are loaded only when the requested analysis runs. It never starts a parser, compiler, model or language server. mode=concept accepts a natural-language query, path and source filters; mode=hybrid uses one natural-language query for exact and local concept evidence, ranks exact evidence first, and retains a bounded semantic supplement. An explicitly enabled semantic judge may classify hybrid candidates, but it is disabled by default and never turns classification into a runtime proof. Slow concept/hybrid requests return status=waiting or running with operationId, progress, and an exact nextRequest using mode=await; copy that request unchanged to continue the same computation. Await expiry never downgrades evidence to literal-only or partial, and final results remain stable for the operation retention window. mode=cancel explicitly stops one operation. allOf and anyOf are explicit literal variants and cannot be mixed with pattern/literal; limit and context are output intent and are never silently dropped. modifiedAfter/modifiedBefore filter worktree files by inclusive/exclusive modification-time bounds in Unix milliseconds. structure requires a nonempty AST pattern and JS/TS/TSX/Go sources; lang is not a field. Outline uses a concrete source file path (not a directory) or retained cursor+matchIndex and follows declared syntax capabilities. imports/tests return bounded static module and related-test candidates without proving runtime execution. validate checks saved source evidence against its recorded origin. Partial coverage stays explicit. ${REQUEST_USAGE_GUIDANCE}`; +export const SIFT_LIGHT_DESCRIPTION = `Search and navigate code with bounded, verifiable evidence. Routine searches should use exact content, filenames or applicable structural modes first. Omitted mode is ordinary exact search and does not load the embedding model; concept/hybrid require vectorSearchEnabled:true in sift-light.json plus an installed model and may take tens of seconds on an uncached scope. Ordinary pattern searches use auto detail/summary; pattern is regex by default and literal=true matches source text exactly. A path selects an existing exact file or root; use mode=files with query to discover an unknown name. scope=strict prevents zero-result path expansion and wholeWord requires word boundaries. mode=capabilities returns a compact names-only project language inventory and the modes available for each detected language; capability providers are loaded only when the requested analysis runs. It never starts a parser, compiler, model or language server. mode=concept accepts a natural-language query, path and source filters; mode=hybrid uses one natural-language query for exact and local concept evidence, ranks exact evidence first, and retains a bounded semantic supplement. An explicitly enabled semantic judge may classify hybrid candidates, but it is disabled by default and never turns classification into a runtime proof. Slow concept/hybrid requests return status=waiting or running with operationId, progress, and an exact nextRequest using mode=await; copy that request unchanged to continue the same computation. Await expiry never downgrades evidence to literal-only or partial, and final results remain stable for the operation retention window. mode=cancel explicitly stops one operation. allOf and anyOf are explicit literal variants and cannot be mixed with pattern/literal; limit and context are output intent and are never silently dropped. modifiedAfter/modifiedBefore filter worktree files by inclusive/exclusive modification-time bounds in Unix milliseconds. structure requires a nonempty AST pattern and JS/TS/TSX/Go sources; lang is not a field. Outline uses a concrete source file path (not a directory) or retained cursor+matchIndex and follows declared syntax capabilities. imports/tests return bounded static module and related-test candidates without proving runtime execution. validate checks saved source evidence against its recorded origin. Partial coverage stays explicit. ${REQUEST_USAGE_GUIDANCE}`; -export const SIFT_LIGHT_MODEL_DESCRIPTION = `Bounded local evidence search. ${MODEL_USAGE_GUIDANCE}. Copy cursors; analysis is evidence, not proof.`; +export const SIFT_LIGHT_MODEL_DESCRIPTION = `Bounded local evidence search. Default exact search is model-free; concept/hybrid require vectorSearchEnabled:true in sift-light.json and a model. ${MODEL_USAGE_GUIDANCE}. Copy cursors; analysis is evidence, not proof.`; export const siftLightSchema = Type.Object({ query: Type.Optional( Type.String({ maxLength: 256, - description: `${fieldGuidance("query")}. Hybrid uses the same query as exact literal text and as the local concept query. Discovery modes preserve their requested path. Concept and hybrid require an explicitly installed local model. A semantic judge is optional and remains disabled unless the active configuration explicitly enables it.`, + description: `${fieldGuidance("query")}. Hybrid uses the same query as exact literal text and as the local concept query. Discovery modes preserve their requested path. Concept and hybrid require vectorSearchEnabled:true in sift-light.json and an explicitly installed local model. A semantic judge is optional and remains disabled unless the active configuration explicitly enables it.`, }), ), scope: Type.Optional(