diff --git a/.changeset/fast-storage-readers.md b/.changeset/fast-storage-readers.md new file mode 100644 index 0000000..e56a72e --- /dev/null +++ b/.changeset/fast-storage-readers.md @@ -0,0 +1,5 @@ +--- +"@ontos-ai/knowhere-sdk": patch +--- + +Read committed parsed-storage result objects concurrently after validating the commit marker. diff --git a/.changeset/parsed-storage-result-layout.md b/.changeset/parsed-storage-result-layout.md deleted file mode 100644 index 984ccf3..0000000 --- a/.changeset/parsed-storage-result-layout.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -'@ontos-ai/knowhere-sdk': patch -'@ontos-ai/knowhere-mcp': patch ---- - -Replace the SDK parsed-document storage adapter with a result-relative object interface and store committed parsed results as expanded Knowhere result files (`manifest.json`, `chunks.json`, optional sidecars, and assets) instead of SDK-specific paged snapshots. - -Remove `assetUrlPolicy` from `knowledge.readChunks` and the MCP `knowhere_read_chunks` schema. SDK remote chunk reads now always request asset URLs from Knowhere, return those remote asset URLs directly on storage misses, and only rewrite asset URLs to storage object URLs when a configured parsed storage already has the corresponding object. diff --git a/.changeset/quiet-pages-reflect.md b/.changeset/quiet-pages-reflect.md new file mode 100644 index 0000000..645d7a5 --- /dev/null +++ b/.changeset/quiet-pages-reflect.md @@ -0,0 +1,5 @@ +--- +"@ontos-ai/knowhere-mcp": patch +--- + +Improve MCP read guidance for page assets and default durable asset URLs for display reads. diff --git a/CHANGELOG.md b/CHANGELOG.md index 59b50fc..fd28af3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 2.1.2 + +### Patch Changes + +- [#115](https://github.com/Ontos-AI/knowhere-node-sdk/pull/115) [`8b7ccca`](https://github.com/Ontos-AI/knowhere-node-sdk/commit/8b7cccaadef54396dcfece2c2ba260e3bf0c20ec) Thanks [@suguanYang](https://github.com/suguanYang)! - Replace the SDK parsed-document storage adapter with a result-relative object interface and store committed parsed results as expanded Knowhere result files (`manifest.json`, `chunks.json`, optional sidecars, and assets) instead of SDK-specific paged snapshots. + + Remove `assetUrlPolicy` from `knowledge.readChunks` and the MCP `knowhere_read_chunks` schema. SDK remote chunk reads now always request asset URLs from Knowhere, return those remote asset URLs directly on storage misses, and only rewrite asset URLs to storage object URLs when a configured parsed storage already has the corresponding object. + ## 2.1.1 ### Patch Changes diff --git a/package.json b/package.json index 4c230c0..37e550f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@ontos-ai/knowhere-sdk", - "version": "2.1.1", + "version": "2.1.2", "description": "Official Node.js SDK for Knowhere document parsing API", "keywords": [ "knowhere", diff --git a/packages/mcp/CHANGELOG.md b/packages/mcp/CHANGELOG.md index f4fc231..2578615 100644 --- a/packages/mcp/CHANGELOG.md +++ b/packages/mcp/CHANGELOG.md @@ -1,5 +1,16 @@ # @ontos-ai/knowhere-mcp +## 2.1.2 + +### Patch Changes + +- [#115](https://github.com/Ontos-AI/knowhere-node-sdk/pull/115) [`8b7ccca`](https://github.com/Ontos-AI/knowhere-node-sdk/commit/8b7cccaadef54396dcfece2c2ba260e3bf0c20ec) Thanks [@suguanYang](https://github.com/suguanYang)! - Replace the SDK parsed-document storage adapter with a result-relative object interface and store committed parsed results as expanded Knowhere result files (`manifest.json`, `chunks.json`, optional sidecars, and assets) instead of SDK-specific paged snapshots. + + Remove `assetUrlPolicy` from `knowledge.readChunks` and the MCP `knowhere_read_chunks` schema. SDK remote chunk reads now always request asset URLs from Knowhere, return those remote asset URLs directly on storage misses, and only rewrite asset URLs to storage object URLs when a configured parsed storage already has the corresponding object. + +- Updated dependencies [[`8b7ccca`](https://github.com/Ontos-AI/knowhere-node-sdk/commit/8b7cccaadef54396dcfece2c2ba260e3bf0c20ec)]: + - @ontos-ai/knowhere-sdk@2.1.2 + ## 2.1.1 ### Patch Changes diff --git a/packages/mcp/README.md b/packages/mcp/README.md index e9c23d9..fb8c84b 100644 --- a/packages/mcp/README.md +++ b/packages/mcp/README.md @@ -182,12 +182,13 @@ When logged in with Read only permission, the MCP server exposes only and makes the parsed document available to outline/read/grep/search tools. - `knowhere_parse_file`: blocking parse for a file path available to the MCP process; waits for completion and makes the parsed document available to - outline/read/grep/search tools. + outline/read/grep/search tools. The path is resolved on the machine running + this stdio MCP server. - `knowhere_async_parse_url`: start parsing a remote URL and return the job immediately. When checking status, use exponential backoff. - `knowhere_async_parse_file`: start parsing a local file path, upload it if needed, and return the job immediately. When checking status, use exponential - backoff. + backoff. The path is resolved on the machine running this stdio MCP server. - `knowhere_async_get_job_status`: check a parse job status. For large PDFs or OCR-heavy files, parsing can take 10+ minutes; poll with `5s`, `10s`, `20s`, `40s`, `80s`, then cap at `120s` between follow-up status checks. After @@ -203,12 +204,14 @@ When logged in with Read only permission, the MCP server exposes only - `knowhere_read_chunks`: read exact chunks from one parsed document by passing `localDocumentId`, published `documentId`, or completed `jobId`. Use `page`/`pageSize` for display reads; asset URLs are returned when the source - or configured storage provides them. + or configured storage provides them. Page screenshots are returned under chunk + `metadata.pageAssets`, not as image chunks. - `knowhere_grep_chunks`: run literal or regex grep over one parsed document by passing `localDocumentId`, published `documentId`, or completed `jobId`. Broad workspace search belongs to `knowhere_search`. - `knowhere_search`: search published documents through the Knowhere API - retrieval query. + retrieval query. Use follow-up `knowhere_read_chunks` calls to inspect page + screenshots under chunk `metadata.pageAssets`. Local-cache tool responses include `document.resultDirectoryPath`; expanded chunks are stored at `/chunks.json` for direct filesystem diff --git a/packages/mcp/package.json b/packages/mcp/package.json index 77e53fc..093ac75 100644 --- a/packages/mcp/package.json +++ b/packages/mcp/package.json @@ -1,6 +1,6 @@ { "name": "@ontos-ai/knowhere-mcp", - "version": "2.1.1", + "version": "2.1.2", "description": "MCP wrapper for the Knowhere Node.js SDK local knowledge tools", "keywords": [ "knowhere", diff --git a/packages/mcp/src/__tests__/mcp.test.ts b/packages/mcp/src/__tests__/mcp.test.ts index a390ef5..e12c450 100644 --- a/packages/mcp/src/__tests__/mcp.test.ts +++ b/packages/mcp/src/__tests__/mcp.test.ts @@ -12,7 +12,12 @@ describe('knowhere MCP wrapper', () => { const toolNames = tools.tools.map((tool) => tool.name).sort(); const statusTool = tools.tools.find((tool) => tool.name === 'knowhere_async_get_job_status'); const listTool = tools.tools.find((tool) => tool.name === 'knowhere_list_documents'); + const parseFileTool = tools.tools.find((tool) => tool.name === 'knowhere_parse_file'); + const asyncParseFileTool = tools.tools.find( + (tool) => tool.name === 'knowhere_async_parse_file', + ); const readTool = tools.tools.find((tool) => tool.name === 'knowhere_read_chunks'); + const searchTool = tools.tools.find((tool) => tool.name === 'knowhere_search'); expect(toolNames).toEqual([ 'knowhere_async_get_job_status', @@ -30,8 +35,16 @@ describe('knowhere MCP wrapper', () => { expect(statusTool?.description).toContain('5s, 10s, 20s, 40s, 80s'); expect(statusTool?.description).toContain('Large PDFs or OCR-heavy files can take 10+ minutes'); expect(listTool?.description).toContain('remote API'); + expect(parseFileTool?.description).toContain( + 'resolved on the machine running this stdio MCP server', + ); + expect(asyncParseFileTool?.description).toContain( + 'resolved on the machine running this stdio MCP server', + ); expect(readTool?.description).toContain('configured parsed storage first'); expect(readTool?.description).toContain('returns asset URLs'); + expect(readTool?.description).toContain('metadata.pageAssets'); + expect(searchTool?.description).toContain('metadata.pageAssets'); await client.close(); await server.close(); }); diff --git a/packages/mcp/src/index.ts b/packages/mcp/src/index.ts index 589760e..4546f34 100644 --- a/packages/mcp/src/index.ts +++ b/packages/mcp/src/index.ts @@ -96,7 +96,7 @@ export async function createKnowhereMcpServer( 'knowhere_parse_file', { description: - 'Blocking parse: submit a local file path available to this MCP process, wait for completion, then make the parsed document available to outline/read/grep/search tools.', + 'Blocking parse: submit a local file path available to this MCP process, wait for completion, then make the parsed document available to outline/read/grep/search tools. The file path is resolved on the machine running this stdio MCP server, not on a remote chat client.', inputSchema: { file: z.string().describe('Local file path available to this MCP server process.'), fileName: z.string().optional(), @@ -150,7 +150,7 @@ export async function createKnowhereMcpServer( 'knowhere_async_parse_file', { description: - 'Start parsing a local file path available to this MCP process, upload it if needed, and return immediately with the parse job. When checking status, call knowhere_async_get_job_status with exponential backoff: 5s, 10s, 20s, 40s, 80s, then cap at 120s. Large PDFs or OCR-heavy files can take 10+ minutes; prefer sparse follow-up status checks over rapid repeated calls. After completion, use the returned localDocumentId, documentId, or jobId with outline/read/grep tools.', + 'Start parsing a local file path available to this MCP process, upload it if needed, and return immediately with the parse job. The file path is resolved on the machine running this stdio MCP server, not on a remote chat client. When checking status, call knowhere_async_get_job_status with exponential backoff: 5s, 10s, 20s, 40s, 80s, then cap at 120s. Large PDFs or OCR-heavy files can take 10+ minutes; prefer sparse follow-up status checks over rapid repeated calls. After completion, use the returned localDocumentId, documentId, or jobId with outline/read/grep tools.', inputSchema: { file: z.string().describe('Local file path available to this MCP server process.'), fileName: z.string().optional(), @@ -243,7 +243,7 @@ export async function createKnowhereMcpServer( 'knowhere_read_chunks', { description: - 'Read exact chunks from one parsed document. Pass localDocumentId, published documentId, or completed jobId. page/pageSize are for display reads and cannot be combined with sectionPath/startChunk/endChunk/chunkId. The SDK reads configured parsed storage first, falls back to remote document chunks for documentId reads, and returns asset URLs when the source or storage provides them.', + 'Read exact chunks from one parsed document. Pass localDocumentId, published documentId, or completed jobId. page/pageSize are for display reads and cannot be combined with sectionPath/startChunk/endChunk/chunkId. The SDK reads configured parsed storage first, falls back to remote document chunks for documentId reads, and returns asset URLs when the source or storage provides them. Page screenshots are returned under chunk metadata.pageAssets, not as image chunks.', inputSchema: { localDocumentId: z.string().optional(), documentId: z.string().optional(), @@ -291,7 +291,7 @@ export async function createKnowhereMcpServer( 'knowhere_search', { description: - 'Search published Knowhere documents with the Knowhere API retrieval query. localDocumentIds only map returned server document IDs back to local cache IDs when available.', + 'Search published Knowhere documents with the Knowhere API retrieval query. localDocumentIds only map returned server document IDs back to local cache IDs when available. Page screenshots are exposed by follow-up read calls under chunk metadata.pageAssets, not as image chunks.', inputSchema: { query: z.string(), namespace: z.string().optional(), diff --git a/src/knowledge/__tests__/knowledge.test.ts b/src/knowledge/__tests__/knowledge.test.ts index 9c834fb..09259bc 100644 --- a/src/knowledge/__tests__/knowledge.test.ts +++ b/src/knowledge/__tests__/knowledge.test.ts @@ -321,6 +321,83 @@ describe('Knowledge', () => { expect(read.totalChunks).toBe(3); }); + it('should read committed parsed result objects concurrently after the commit gate', async () => { + const cacheDirectory = await createTempDirectory(); + const parseResult = createParseResult(); + const storage = createInMemoryParsedStorage(); + const readEvents: string[] = []; + const delayedResultPaths = new Set(['manifest.json', 'chunks.json', 'doc_nav.json']); + const releaseByPath = new Map void>(); + let shouldDelayResultReads = true; + + storage.seedResult({ + documentId: 'doc_remote', + revisionKey: 'jres_remote', + result: parseResult, + }); + + const readObject = storage.readObject.bind(storage); + storage.readObject = async ( + params: ParsedDocumentObjectParams, + ): Promise => { + readEvents.push(`start:${params.path}`); + if (delayedResultPaths.has(params.path) && shouldDelayResultReads) { + await new Promise((resolve) => { + releaseByPath.set(params.path, resolve); + }); + } + + const object = await readObject(params); + readEvents.push(`finish:${params.path}`); + return object; + }; + + const releaseDelayedReads = (): void => { + shouldDelayResultReads = false; + for (const release of releaseByPath.values()) { + release(); + } + releaseByPath.clear(); + }; + + const { client, documentsListChunks } = createClient(parseResult); + const knowledge = new Knowledge(client, { cacheDirectory }).withParsedStorage({ storage }); + const readPromise = knowledge.readChunks({ + documentId: 'doc_remote', + revisionKey: 'jres_remote', + page: 1, + pageSize: 2, + }); + + try { + await new Promise((resolve) => { + setImmediate(resolve); + }); + + const commitFinishIndex = readEvents.indexOf('finish:.knowhere-sdk/commit.json'); + const resultStartIndexes = ['manifest.json', 'chunks.json', 'doc_nav.json'].map( + (objectPath) => readEvents.indexOf(`start:${objectPath}`), + ); + + expect(commitFinishIndex).toBeGreaterThanOrEqual(0); + expect(resultStartIndexes.every((index) => index > commitFinishIndex)).toBe(true); + expect(readEvents.filter((event) => event.startsWith('start:'))).toEqual([ + 'start:.knowhere-sdk/commit.json', + 'start:manifest.json', + 'start:chunks.json', + 'start:doc_nav.json', + ]); + + releaseDelayedReads(); + const read = await readPromise; + + expect(documentsListChunks).not.toHaveBeenCalled(); + expect(read.chunks.map((chunk) => chunk.chunkId)).toEqual(['chunk-intro', 'chunk-table']); + } finally { + releaseDelayedReads(); + } + }); + it('should hydrate stored page asset URLs from snake_case chunks metadata', async () => { const cacheDirectory = await createTempDirectory(); const storage = createInMemoryParsedStorage(); diff --git a/src/knowledge/knowledge.ts b/src/knowledge/knowledge.ts index 620ca7b..7dde3b0 100644 --- a/src/knowledge/knowledge.ts +++ b/src/knowledge/knowledge.ts @@ -1081,16 +1081,23 @@ export class Knowledge { return null; } - const manifestObject = await parsedStorageConfig.storage.readObject({ - documentId, - revisionKey, - path: 'manifest.json', - }); - const chunksObject = await parsedStorageConfig.storage.readObject({ - documentId, - revisionKey, - path: 'chunks.json', - }); + const [manifestObject, chunksObject, docNavObject] = await Promise.all([ + parsedStorageConfig.storage.readObject({ + documentId, + revisionKey, + path: 'manifest.json', + }), + parsedStorageConfig.storage.readObject({ + documentId, + revisionKey, + path: 'chunks.json', + }), + parsedStorageConfig.storage.readObject({ + documentId, + revisionKey, + path: 'doc_nav.json', + }), + ]); if (!manifestObject || !chunksObject) { return null; } @@ -1100,11 +1107,6 @@ export class Knowledge { payload: readJsonObject(chunksObject), sourceFileName: manifest.sourceFileName, }); - const docNavObject = await parsedStorageConfig.storage.readObject({ - documentId, - revisionKey, - path: 'doc_nav.json', - }); const docNav = docNavObject ? readJsonObject<{ readonly sections?: DocNavSection[] }>(docNavObject) : undefined;