From 5ca2f6786d37eed6906ede1d45f8702470fc0ffb Mon Sep 17 00:00:00 2001 From: TheCryptoDonkey Date: Wed, 23 Sep 2026 07:01:13 +0100 Subject: [PATCH] feat: build navigation on first use, read whole files and merge packet ranges The repeated Pro run showed Context's input following its extra turns. The index now builds on the first search, explore or coverage call; a build source without startLine or endLine reads from the start or to the end; overlapping ranges merge instead of failing. --- docs/DAILY-USE.md | 5 ++- docs/LOCAL-NAVIGATION.md | 6 ++- docs/WORKER-PACKETS.md | 7 +-- .../src/repository-navigation-mcp.test.ts | 28 ++++++++++-- .../src/repository-navigation-mcp.ts | 43 +++++++++++++++---- .../src/repository-navigation.ts | 5 +++ .../src/repository-packet-mcp.test.ts | 16 +++++++ 7 files changed, 92 insertions(+), 18 deletions(-) diff --git a/docs/DAILY-USE.md b/docs/DAILY-USE.md index 96451f2..4a5a0a0 100644 --- a/docs/DAILY-USE.md +++ b/docs/DAILY-USE.md @@ -14,8 +14,9 @@ For explicit client exports, use the [offline daily usage importer](DAILY-USAGE. to produce a private request receipt and task summary. It preserves missing coverage and reports observed usage rather than claiming savings. -1. Call `repository_status`. If freshness is `unavailable`, `stale` or - `unknown`, call `repository_refresh`. `current` only covers the bounded, +1. Call `repository_status` and confirm its root. If freshness is `stale` or + `unknown`, call `repository_refresh`; an `unavailable` index is built by the + first search, explore or coverage call. `current` only covers the bounded, allowlisted manifest, not every file in the repository. 2. Search for one exact ASCII identifier, starting small: diff --git a/docs/LOCAL-NAVIGATION.md b/docs/LOCAL-NAVIGATION.md index c32bc37..d9f27d4 100644 --- a/docs/LOCAL-NAVIGATION.md +++ b/docs/LOCAL-NAVIGATION.md @@ -69,9 +69,11 @@ node packages/context-tools/bin/encrypted-context.mjs navigate /absolute/reposit ``` There is no identity or encrypted-cache argument. Each process owns its own -index. Call `repository_refresh` before searching and again after source edits; +index. Search, explore and coverage build it on first use (concurrent first +calls share one build); call `repository_refresh` again after source edits. `repository_status` reports the generation, revision, exclusion counts and -freshness. Refresh explicitly whenever freshness is `stale` or `unknown`. +freshness without building. Refresh explicitly whenever freshness is `stale` or +`unknown`: a built index is never replaced without a refresh call. Explore one symbol with `repository_explore`, for example `RepositoryNavigation` or `RepositoryNavigation.search`, or search for one identifier with `repository_search`. diff --git a/docs/WORKER-PACKETS.md b/docs/WORKER-PACKETS.md index feea28f..3d668dc 100644 --- a/docs/WORKER-PACKETS.md +++ b/docs/WORKER-PACKETS.md @@ -58,9 +58,10 @@ then call `repository_refresh` and retain its `generation`. Replace the example paths and anchors with located evidence. Use `mode: "build"` and `{path, startLine, endLine}` source entries for reviewed exact ranges, including languages without syntax planning. Through MCP only `sources` is -required: omitted handoff metadata gets a neutral task and acceptance check, and -an `endLine` past the end of a file reads to its last line (the packet records -the range actually read). A range starting past the end is still rejected with +required: omitted handoff metadata gets a neutral task and acceptance check. A +build source without `startLine` starts at line 1; without `endLine`, or with one +past the end of the file, it reads to the last line; overlapping or adjacent +ranges in one file are merged. The packet records the ranges actually read. A range starting past the end is still rejected with the file's line count. A plan anchor outside every supported block is rejected with a pointer to `mode: "build"`. The CLI keeps the strict spec. The server accepts an inline spec; it cannot accept a different root, spec-file path, output path or shell command. diff --git a/packages/context-tools/src/repository-navigation-mcp.test.ts b/packages/context-tools/src/repository-navigation-mcp.test.ts index a84c13f..6c77674 100644 --- a/packages/context-tools/src/repository-navigation-mcp.test.ts +++ b/packages/context-tools/src/repository-navigation-mcp.test.ts @@ -79,7 +79,25 @@ describe('repository navigation MCP adapter', () => { } }) - it('reports null generation before refresh, errors on early search, then serves after refresh', async () => { + it('builds once when several first calls arrive together', async () => { + const root = await makeRoot() + const { client, serverClose } = await connect(root) + try { + const results = await Promise.all([ + client.callTool({ name: 'repository_search', arguments: { term: 'alphaToken' } }), + client.callTool({ name: 'repository_search', arguments: { term: 'alphaToken' } }), + client.callTool({ name: 'repository_explore', arguments: { symbol: 'alphaToken' } }), + ]) as Array<{ isError?: boolean; content: Array<{ type: string; text: string }> }> + for (const result of results) expect(result.isError, textOf(result)).not.toBe(true) + const generations = new Set(results.map((result) => /generation ([0-9a-f-]{36})/.exec(textOf(result))?.[1])) + expect(generations.size).toBe(1) + expect(generations.has(undefined)).toBe(false) + } finally { + await serverClose() + } + }) + + it('reports null generation before first use, builds on an early search, then serves after refresh', async () => { const root = await makeRoot() const { client, serverClose } = await connect(root) try { @@ -103,7 +121,11 @@ describe('repository navigation MCP adapter', () => { name: 'repository_search', arguments: { term: 'alphaToken' }, })) as { content: Array<{ type: string; text: string }>; isError?: boolean } - expect(early.isError).toBe(true) + expect(early.isError).not.toBe(true) + expect(textOf(early)).toMatch(/alphaToken|alphatoken/) + const built = JSON.parse(textOf(await client.callTool({ name: 'repository_status', arguments: {} }) as { content: Array<{ type: string; text: string }> })) as { freshness: string; generation: string | null } + expect(built.freshness).toBe('current') + expect(built.generation).not.toBeNull() const refreshed = (await client.callTool({ name: 'repository_refresh', @@ -400,7 +422,7 @@ describe('repository navigation MCP compact rendering and explore', () => { const { client, serverClose } = await connect(root) try { const early = (await client.callTool({ name: 'repository_explore', arguments: { symbol: 'ownerOptions' } })) as { isError?: boolean } - expect(early.isError).toBe(true) + expect(early.isError).not.toBe(true) const refreshed = JSON.parse(textOf(await client.callTool({ name: 'repository_refresh', arguments: {} }) as { content: unknown })) as { generation: string } const explored = (await client.callTool({ name: 'repository_explore', arguments: { symbol: 'ownerOptions', expectedGeneration: refreshed.generation } })) as { content: Array<{ type: string; text: string }>; isError?: boolean } expect(explored.isError).not.toBe(true) diff --git a/packages/context-tools/src/repository-navigation-mcp.ts b/packages/context-tools/src/repository-navigation-mcp.ts index d4ef23b..d49b8d5 100644 --- a/packages/context-tools/src/repository-navigation-mcp.ts +++ b/packages/context-tools/src/repository-navigation-mcp.ts @@ -30,9 +30,22 @@ const packetMetadataSchema = z.object({ exclusions: metadataStrings.default([]), unresolvedQuestions: metadataStrings.default([]), }).strict() +// A source without startLine starts at line 1; without endLine it reads to the end +// of the file. Overlapping or adjacent ranges in one file are merged. const packetBuildSpecSchema = packetMetadataSchema.extend({ - sources: z.array(z.object({ path: z.string().min(1), startLine: z.number().int().min(1), endLine: z.number().int().min(1) }).strict()).max(32), -}).strict() + sources: z.array(z.object({ path: z.string().min(1), startLine: z.number().int().min(1).optional(), endLine: z.number().int().min(1).optional() }).strict()).max(32), +}).strict().transform((spec) => ({ ...spec, sources: mergeRanges(spec.sources.map((source) => ({ path: source.path, startLine: source.startLine ?? 1, endLine: source.endLine ?? Number.MAX_SAFE_INTEGER }))) })) + +function mergeRanges(sources: Array<{ path: string; startLine: number; endLine: number }>): Array<{ path: string; startLine: number; endLine: number }> { + const sorted = [...sources].sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0) || a.startLine - b.startLine) + const merged: Array<{ path: string; startLine: number; endLine: number }> = [] + for (const source of sorted) { + const last = merged.at(-1) + if (last && last.path === source.path && source.startLine <= last.endLine + 1) last.endLine = Math.max(last.endLine, source.endLine) + else merged.push({ ...source }) + } + return merged +} const packetPlanSpecSchema = packetMetadataSchema.extend({ sources: z.array(z.object({ path: z.string().min(1), line: z.number().int().min(1) }).strict()).max(32), }).strict() @@ -95,8 +108,8 @@ export function createRepositoryNavigationServer(root: string): RepositoryNaviga { name: 'repository-navigation', version: '0.0.0' }, { instructions: - 'Read-only navigation of one local repository. Call repository_refresh first ' + - 'and after edits or branch changes. For a symbol, call repository_explore, then ' + + 'Read-only navigation of one local repository. The index builds on first use; ' + + 'call repository_refresh after edits or branch changes. For a symbol, call repository_explore, then ' + 'repository_packet for exact source. Use repository_search only for literals, ' + 'narrowed with pathPrefix. Before submitting, check the draft with ' + 'repository_coverage. Matching is exact-token, not semantic, so absence is not ' + @@ -104,6 +117,17 @@ export function createRepositoryNavigationServer(root: string): RepositoryNaviga }, ) + // Search, explore and coverage build the index on first use rather than failing, + // which saved a status and refresh call in every recorded session. Later + // staleness still needs an explicit refresh, so a generation never changes + // under a caller without its asking. + let firstBuild: Promise | null = null + const buildOnFirstUse = async (signal?: AbortSignal): Promise => { + if (navigation.built) return + firstBuild ??= navigation.refresh(signal).finally(() => { firstBuild = null }) + await firstBuild + } + server.registerTool( 'repository_status', { @@ -126,8 +150,8 @@ export function createRepositoryNavigationServer(root: string): RepositoryNaviga 'repository_refresh', { description: - 'Rebuild the in-memory index. Call before first use and after source changes; ' + - 'invalidates search cursors.', + 'Rebuild the in-memory index after source changes (search, explore and coverage ' + + 'build it on first use); invalidates search cursors.', inputSchema: z.object({}).strict(), annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false }, }, @@ -154,6 +178,7 @@ export function createRepositoryNavigationServer(root: string): RepositoryNaviga }, async (input, extra) => { try { + await buildOnFirstUse(extra.signal) if (input.expectedGeneration !== undefined) { const status = await navigation.status(extra.signal) if (status.generation !== input.expectedGeneration) throw new Error('repository explore expectedGeneration does not match current navigation') @@ -184,6 +209,7 @@ export function createRepositoryNavigationServer(root: string): RepositoryNaviga const symbols = distinct.slice(0, COVERAGE_MAX_SYMBOLS) const symbolsNotChecked = distinct.slice(COVERAGE_MAX_SYMBOLS) if (symbols.length === 0 && !input.evidence?.length) throw new Error('repository coverage needs symbols, evidence or both') + await buildOnFirstUse(extra.signal) const explored: ExploreResult[] = [] for (const symbol of symbols) { const result = await navigation.explore({ symbol, pathPrefix: input.pathPrefix }, extra.signal) @@ -228,6 +254,7 @@ export function createRepositoryNavigationServer(root: string): RepositoryNaviga async (input, extra) => { try { const { format, ...options } = input + await buildOnFirstUse(extra.signal) const result = await navigation.search(options, extra.signal) return { content: [{ type: 'text' as const, text: format === 'json' ? JSON.stringify(result) : renderSearch(result) }] } } catch (error) { @@ -240,8 +267,8 @@ export function createRepositoryNavigationServer(root: string): RepositoryNaviga 'repository_packet', { description: - 'Verbatim source. mode build: exact ranges (sources path, startLine, endLine; ' + - 'an endLine past the end reads to the end). ' + + 'Verbatim source. mode build: ranges (sources path, startLine, endLine; omit ' + + 'endLine or pass one past the end to read to the end). ' + 'mode plan: complete TypeScript/JavaScript blocks around anchors (sources path, ' + 'line). Pass the current generation; stale navigation is rejected. Capped at ' + 'maxBytes (64 KiB default).', diff --git a/packages/context-tools/src/repository-navigation.ts b/packages/context-tools/src/repository-navigation.ts index 08c3043..2ec635e 100644 --- a/packages/context-tools/src/repository-navigation.ts +++ b/packages/context-tools/src/repository-navigation.ts @@ -436,6 +436,11 @@ export class RepositoryNavigation { this.rootInput = root; } + /** Whether any generation has been built; cheap, unlike status(). */ + get built(): boolean { + return this.generation !== null; + } + async status(signal?: AbortSignal): Promise { throwIfAborted(signal); const gen = this.generation; diff --git a/packages/context-tools/src/repository-packet-mcp.test.ts b/packages/context-tools/src/repository-packet-mcp.test.ts index eaaa6ba..17f32e6 100644 --- a/packages/context-tools/src/repository-packet-mcp.test.ts +++ b/packages/context-tools/src/repository-packet-mcp.test.ts @@ -254,6 +254,22 @@ describe('repository packet MCP adapter', () => { } finally { await connection.close() } }) + it('reads a whole file from a path alone and merges overlapping ranges', async () => { + const value = await root(); const connection = await connect(value) + try { + const refreshed = await connection.client.callTool({ name: 'repository_refresh', arguments: {} }) as { content: unknown } + const generation = (JSON.parse(text(refreshed)) as { generation: string }).generation + const whole = await call(connection.client, { mode: 'build', spec: { sources: [{ path: 'alpha.ts' }] }, expectedGeneration: generation }) + expect(whole.isError).not.toBe(true) + expect(JSON.parse(text(whole)).packet.sources).toMatchObject([{ path: 'alpha.ts', startLine: 1, endLine: 3 }]) + const overlapping = await call(connection.client, { mode: 'build', spec: { sources: [{ path: 'alpha.ts', startLine: 2, endLine: 3 }, { path: 'alpha.ts', startLine: 1, endLine: 2 }] }, expectedGeneration: generation }) + expect(overlapping.isError).not.toBe(true) + const packet = JSON.parse(text(overlapping)).packet + expect(packet.sources).toMatchObject([{ path: 'alpha.ts', startLine: 1, endLine: 3 }]) + expect(packet.originalSpec.sources).toEqual([{ path: 'alpha.ts', startLine: 1, endLine: 3 }]) + } finally { await connection.close() } + }) + it('points a plan anchor outside any block at an exact build range', async () => { const value = await root() await writeFile(join(value, 'beta.ts'), "import { alpha } from './alpha'\nexport function beta() {\n return alpha()\n}\n")