diff --git a/src/prompts/index.ts b/src/prompts/index.ts index 34f76b4..f27c10c 100644 --- a/src/prompts/index.ts +++ b/src/prompts/index.ts @@ -1,9 +1,11 @@ import * as path from 'node:path' // Still needed for path.relative for file map display logic import * as vscode from 'vscode' // For vscode.workspace.fs and vscode.Uri import type { VscodeTreeItem } from '../types' -import { isBinaryFile } from '../utils/file-system' +import { readTextFileForContext } from '../utils/file-system' import { XML_FORMATTING_INSTRUCTIONS } from './xml-instruction' +const MAX_PARALLEL_CONTEXT_READS = 8 + // Helper function moved to module scope function hasSelectedDescendant( item: VscodeTreeItem, @@ -195,6 +197,55 @@ function buildTreeString( } } +async function readSelectedFileContent(uriString: string): Promise { + const fileUri = vscode.Uri.parse(uriString) + try { + const result = await readTextFileForContext(fileUri) + if (result.type === 'text') { + return `File: ${fileUri.fsPath}\n\`\`\`\n${result.content ?? ''}\n\`\`\`\n\n` + } + if (result.type === 'binary') { + console.log('Skipping binary file:', fileUri.fsPath) + return `File: ${fileUri.fsPath}\n*** Skipped: Binary file ***\n\n` + } + + console.log('Not a file (possibly a directory):', fileUri.fsPath) + return '' + } catch (error: unknown) { + let errorMessage = 'Unknown error' + if (error instanceof Error) { + errorMessage = error.message + } else if (typeof error === 'string') { + errorMessage = error + } + console.warn( + `Could not read file ${fileUri.fsPath} for context: ${errorMessage}`, + ) + return `File: ${fileUri.fsPath}\n*** Error reading file: ${errorMessage} ***\n\n` + } +} + +async function mapWithConcurrency( + items: T[], + limit: number, + mapper: (item: T) => Promise, +): Promise { + const results = new Array(items.length) + let nextIndex = 0 + const workerCount = Math.min(limit, items.length) + + await Promise.all( + Array.from({ length: workerCount }, async () => { + while (nextIndex < items.length) { + const currentIndex = nextIndex++ + results[currentIndex] = await mapper(items[currentIndex]!) + } + }), + ) + + return results +} + /** * Generates the file contents string for selected files. * @param selectedUris - A set of selected URI strings. @@ -203,49 +254,16 @@ function buildTreeString( export async function generateFileContents( selectedUris: Set, ): Promise { - let contentsStr = '' // Sort URI strings for consistent order. fsPath might be better for sorting if paths are complex. const sortedUriStrings = Array.from(selectedUris).sort() - for (const uriString of sortedUriStrings) { - const fileUri = vscode.Uri.parse(uriString) - try { - // Ensure it's a file using vscode.workspace.fs.stat - const stat = await vscode.workspace.fs.stat(fileUri) - if (stat.type === vscode.FileType.File) { - // Check if the file is binary and skip it - const isBinary = await isBinaryFile(fileUri) - if (isBinary) { - console.log('Skipping binary file:', fileUri.fsPath) - contentsStr += `File: ${fileUri.fsPath}\n*** Skipped: Binary file ***\n\n` - continue - } - - const contentBuffer = await vscode.workspace.fs.readFile(fileUri) - const content = Buffer.from(contentBuffer).toString('utf8') - // Use full fsPath in the header as per user's example - contentsStr += `File: ${fileUri.fsPath}\n\`\`\`\n${content}\n\`\`\`\n\n` - } else { - // This case should ideally not happen if only files are in selectedUris for contents - console.log('Not a file (possibly a directory):', fileUri.fsPath) - } - } catch (error: unknown) { - let errorMessage = 'Unknown error' - if (error instanceof Error) { - errorMessage = error.message - } else if (typeof error === 'string') { - errorMessage = error - } - console.warn( - `Could not read file ${fileUri.fsPath} for context: ${errorMessage}`, - ) - // Add a note about the missing/unreadable file in the context - contentsStr += `File: ${fileUri.fsPath}\n*** Error reading file: ${errorMessage} ***\n\n` - } - } + const chunks = await mapWithConcurrency( + sortedUriStrings, + MAX_PARALLEL_CONTEXT_READS, + readSelectedFileContent, + ) - // Trim the trailing newlines - return contentsStr.trim() + return chunks.join('').trim() } /** diff --git a/src/test/suite/utils/file-system.test.ts b/src/test/suite/utils/file-system.test.ts index 4a65746..1c06c81 100644 --- a/src/test/suite/utils/file-system.test.ts +++ b/src/test/suite/utils/file-system.test.ts @@ -8,10 +8,12 @@ import { MAX_LIST_FILES, MAX_TREE_DEPTH, MAX_TREE_NODES, + clearIgnoreContextCache, getDirectoryChildren, getWorkspaceRoots, listFilesUnderUri, loadIgnoreRulesForRoot, + readTextFileForContext, } from '../../../utils/file-system' // Mirrors the heavy-dir defaults the provider passes (see FileExplorerProvider). @@ -99,6 +101,41 @@ describe('file-system: loadIgnoreRulesForRoot', () => { }) }) +describe('file-system: readTextFileForContext', () => { + before(async () => { + await fsp.rm(baseDir, { recursive: true, force: true }) + await fsp.mkdir(baseDir, { recursive: true }) + await writeFixture('plain.txt', 'hello') + await writeFixture('image.png', 'not actually png, but extension is binary') + await fsp.mkdir(path.join(baseDir, 'folder'), { recursive: true }) + }) + after(async () => { + await fsp.rm(baseDir, { recursive: true, force: true }) + }) + + it('reads text files', async () => { + const result = await readTextFileForContext( + vscode.Uri.file(path.join(baseDir, 'plain.txt')), + ) + assert.strictEqual(result.type, 'text') + assert.strictEqual(result.content, 'hello') + }) + + it('skips known binary extensions before reading content as text', async () => { + const result = await readTextFileForContext( + vscode.Uri.file(path.join(baseDir, 'image.png')), + ) + assert.strictEqual(result.type, 'binary') + }) + + it('reports directories as not-file', async () => { + const result = await readTextFileForContext( + vscode.Uri.file(path.join(baseDir, 'folder')), + ) + assert.strictEqual(result.type, 'not-file') + }) +}) + describe('file-system: abort handling (no workspace required)', () => { it('getDirectoryChildren rejects when the signal is already aborted', async () => { const controller = new AbortController() @@ -129,6 +166,7 @@ describe('file-system: lazy tree APIs (workspace-scoped)', () => { let restore: (() => void) | null = null before(async function () { + clearIgnoreContextCache() await fsp.rm(baseDir, { recursive: true, force: true }) await fsp.mkdir(baseDir, { recursive: true }) await writeFixture('src/app.ts', 'export const a = 1') @@ -147,6 +185,7 @@ describe('file-system: lazy tree APIs (workspace-scoped)', () => { after(async () => { restore?.() + clearIgnoreContextCache() await fsp.rm(baseDir, { recursive: true, force: true }) }) @@ -207,4 +246,18 @@ describe('file-system: lazy tree APIs (workspace-scoped)', () => { 'no gitignored files', ) }) + + it('clearIgnoreContextCache lets callers pick up changed ignore rules', async () => { + await writeFixture('.gitignore', '') + clearIgnoreContextCache() + + const parent = vscode.Uri.file(baseDir).toString() + const { roots: children } = await getDirectoryChildren( + parent, + DEFAULT_EXCLUDES, + ) + const labels = children.map((c) => c.label) + + assert.ok(labels.includes('ignored'), 'updated gitignore rules applied') + }) }) diff --git a/src/utils/file-system.ts b/src/utils/file-system.ts index 03b1b27..5581e51 100644 --- a/src/utils/file-system.ts +++ b/src/utils/file-system.ts @@ -126,6 +126,11 @@ export interface FileTreeResult { truncated: boolean } +export interface TextFileReadResult { + type: 'text' | 'binary' | 'not-file' + content?: string +} + interface IgnoreContext { ign: ignore.Ignore userIgnore: ignore.Ignore @@ -137,6 +142,16 @@ interface ScanBudget { truncated: boolean } +const IGNORE_CONTEXT_CACHE_TTL_MS = 10_000 +const ignoreContextCache = new Map< + string, + { + expiresAt: number + context?: IgnoreContext + pending?: Promise + } +>() + function throwIfAborted(signal?: AbortSignal): void { if (signal?.aborted) { const err = new Error('Aborted') @@ -222,6 +237,13 @@ export function looksBinary(chunk: Uint8Array): boolean { return analyzeByteContent(chunk) } +export function hasBinaryFileExtension(uri: vscode.Uri): boolean { + const fileName = uri.path.toLowerCase() + const lastDot = fileName.lastIndexOf('.') + if (lastDot === -1) return false + return BINARY_EXTENSIONS.has(fileName.substring(lastDot)) +} + export async function isBinaryFile(uri: vscode.Uri): Promise { try { const stats = await vscode.workspace.fs.stat(uri) @@ -229,9 +251,7 @@ export async function isBinaryFile(uri: vscode.Uri): Promise { return false } - const fileName = uri.path.toLowerCase() - const extension = fileName.substring(fileName.lastIndexOf('.')) - if (BINARY_EXTENSIONS.has(extension)) { + if (hasBinaryFileExtension(uri)) { return true } @@ -243,6 +263,30 @@ export async function isBinaryFile(uri: vscode.Uri): Promise { } } +export async function readTextFileForContext( + uri: vscode.Uri, +): Promise { + const stat = await vscode.workspace.fs.stat(uri) + if (stat.type !== vscode.FileType.File) { + return { type: 'not-file' } + } + + if (hasBinaryFileExtension(uri)) { + return { type: 'binary' } + } + + const contentBuffer = await vscode.workspace.fs.readFile(uri) + const chunk = contentBuffer.slice(0, 8000) + if (looksBinary(chunk)) { + return { type: 'binary' } + } + + return { + type: 'text', + content: Buffer.from(contentBuffer).toString('utf8'), + } +} + function resolveExcludesPath(input: string): string { let p = input.trim() if (p.startsWith('~')) { @@ -353,6 +397,22 @@ function buildIgnoreContext( return { ign, userIgnore, rootUri } } +export function clearIgnoreContextCache(): void { + ignoreContextCache.clear() +} + +function getIgnoreContextCacheKey( + rootUri: vscode.Uri, + excludedDirs: string[], + useGitignore: boolean, +): string { + return JSON.stringify({ + root: rootUri.toString(), + useGitignore, + excludedDirs, + }) +} + function findWorkspaceFolderForUri( uri: vscode.Uri, ): vscode.WorkspaceFolder | undefined { @@ -378,8 +438,38 @@ async function createIgnoreContextForUri( throw new Error('No workspace folder for URI') } const useGitignore = options?.useGitignore !== false - const lines = await loadIgnoreRulesForRoot(folder.uri.fsPath, useGitignore) - return buildIgnoreContext(folder.uri, excludedDirs, lines) + const cacheKey = getIgnoreContextCacheKey( + folder.uri, + excludedDirs, + useGitignore, + ) + const cached = ignoreContextCache.get(cacheKey) + if (cached?.context && cached.expiresAt > Date.now()) { + return cached.context + } + if (cached?.pending) { + return cached.pending + } + + const pending = loadIgnoreRulesForRoot(folder.uri.fsPath, useGitignore) + .then((lines) => buildIgnoreContext(folder.uri, excludedDirs, lines)) + .then((context) => { + ignoreContextCache.set(cacheKey, { + expiresAt: Date.now() + IGNORE_CONTEXT_CACHE_TTL_MS, + context, + }) + return context + }) + .catch((error) => { + ignoreContextCache.delete(cacheKey) + throw error + }) + + ignoreContextCache.set(cacheKey, { + expiresAt: 0, + pending, + }) + return pending } function shouldIgnorePath(