perf: speed up context gathering

src/prompts/index.ts

@@ -1,9 +1,11 @@
 import * as path from 'node:path' // Still needed for path.relative for file map display logic
 import * as vscode from 'vscode' // For vscode.workspace.fs and vscode.Uri
 import type { VscodeTreeItem } from '../types'
-import { isBinaryFile } from '../utils/file-system'
+import { readTextFileForContext } from '../utils/file-system'
 import { XML_FORMATTING_INSTRUCTIONS } from './xml-instruction'
 
+const MAX_PARALLEL_CONTEXT_READS = 8
+
 // Helper function moved to module scope
 function hasSelectedDescendant(
 	item: VscodeTreeItem,
@@ -195,6 +197,55 @@ function buildTreeString(
 	}
 }
 
+async function readSelectedFileContent(uriString: string): Promise<string> {
+	const fileUri = vscode.Uri.parse(uriString)
+	try {
+		const result = await readTextFileForContext(fileUri)
+		if (result.type === 'text') {
+			return `File: ${fileUri.fsPath}\n\`\`\`\n${result.content ?? ''}\n\`\`\`\n\n`
+		}
+		if (result.type === 'binary') {
+			console.log('Skipping binary file:', fileUri.fsPath)
+			return `File: ${fileUri.fsPath}\n*** Skipped: Binary file ***\n\n`
+		}
+
+		console.log('Not a file (possibly a directory):', fileUri.fsPath)
+		return ''
+	} catch (error: unknown) {
+		let errorMessage = 'Unknown error'
+		if (error instanceof Error) {
+			errorMessage = error.message
+		} else if (typeof error === 'string') {
+			errorMessage = error
+		}
+		console.warn(
+			`Could not read file ${fileUri.fsPath} for context: ${errorMessage}`,
+		)
+		return `File: ${fileUri.fsPath}\n*** Error reading file: ${errorMessage} ***\n\n`
+	}
+}
+
+async function mapWithConcurrency<T, R>(
+	items: T[],
+	limit: number,
+	mapper: (item: T) => Promise<R>,
+): Promise<R[]> {
+	const results = new Array<R>(items.length)
+	let nextIndex = 0
+	const workerCount = Math.min(limit, items.length)
+
+	await Promise.all(
+		Array.from({ length: workerCount }, async () => {
+			while (nextIndex < items.length) {
+				const currentIndex = nextIndex++
+				results[currentIndex] = await mapper(items[currentIndex]!)
+			}
+		}),
+	)
+
+	return results
+}
+
 /**
  * Generates the file contents string for selected files.
  * @param selectedUris - A set of selected URI strings.
@@ -203,49 +254,16 @@ function buildTreeString(
 export async function generateFileContents(
 	selectedUris: Set<string>,
 ): Promise<string> {
-	let contentsStr = ''
 	// Sort URI strings for consistent order. fsPath might be better for sorting if paths are complex.
 	const sortedUriStrings = Array.from(selectedUris).sort()
 
-	for (const uriString of sortedUriStrings) {
-		const fileUri = vscode.Uri.parse(uriString)
-		try {
-			// Ensure it's a file using vscode.workspace.fs.stat
-			const stat = await vscode.workspace.fs.stat(fileUri)
-			if (stat.type === vscode.FileType.File) {
-				// Check if the file is binary and skip it
-				const isBinary = await isBinaryFile(fileUri)
-				if (isBinary) {
-					console.log('Skipping binary file:', fileUri.fsPath)
-					contentsStr += `File: ${fileUri.fsPath}\n*** Skipped: Binary file ***\n\n`
-					continue
-				}
-
-				const contentBuffer = await vscode.workspace.fs.readFile(fileUri)
-				const content = Buffer.from(contentBuffer).toString('utf8')
-				// Use full fsPath in the header as per user's example
-				contentsStr += `File: ${fileUri.fsPath}\n\`\`\`\n${content}\n\`\`\`\n\n`
-			} else {
-				// This case should ideally not happen if only files are in selectedUris for contents
-				console.log('Not a file (possibly a directory):', fileUri.fsPath)
-			}
-		} catch (error: unknown) {
-			let errorMessage = 'Unknown error'
-			if (error instanceof Error) {
-				errorMessage = error.message
-			} else if (typeof error === 'string') {
-				errorMessage = error
-			}
-			console.warn(
-				`Could not read file ${fileUri.fsPath} for context: ${errorMessage}`,
-			)
-			// Add a note about the missing/unreadable file in the context
-			contentsStr += `File: ${fileUri.fsPath}\n*** Error reading file: ${errorMessage} ***\n\n`
-		}
-	}
+	const chunks = await mapWithConcurrency(
+		sortedUriStrings,
+		MAX_PARALLEL_CONTEXT_READS,
+		readSelectedFileContent,
+	)
 
-	// Trim the trailing newlines
-	return contentsStr.trim()
+	return chunks.join('').trim()
 }
 
 /**

src/test/suite/utils/file-system.test.ts

@@ -8,10 +8,12 @@ import {
 	MAX_LIST_FILES,
 	MAX_TREE_DEPTH,
 	MAX_TREE_NODES,
+	clearIgnoreContextCache,
 	getDirectoryChildren,
 	getWorkspaceRoots,
 	listFilesUnderUri,
 	loadIgnoreRulesForRoot,
+	readTextFileForContext,
 } from '../../../utils/file-system'
 
 // Mirrors the heavy-dir defaults the provider passes (see FileExplorerProvider).
@@ -99,6 +101,41 @@ describe('file-system: loadIgnoreRulesForRoot', () => {
 	})
 })
 
+describe('file-system: readTextFileForContext', () => {
+	before(async () => {
+		await fsp.rm(baseDir, { recursive: true, force: true })
+		await fsp.mkdir(baseDir, { recursive: true })
+		await writeFixture('plain.txt', 'hello')
+		await writeFixture('image.png', 'not actually png, but extension is binary')
+		await fsp.mkdir(path.join(baseDir, 'folder'), { recursive: true })
+	})
+	after(async () => {
+		await fsp.rm(baseDir, { recursive: true, force: true })
+	})
+
+	it('reads text files', async () => {
+		const result = await readTextFileForContext(
+			vscode.Uri.file(path.join(baseDir, 'plain.txt')),
+		)
+		assert.strictEqual(result.type, 'text')
+		assert.strictEqual(result.content, 'hello')
+	})
+
+	it('skips known binary extensions before reading content as text', async () => {
+		const result = await readTextFileForContext(
+			vscode.Uri.file(path.join(baseDir, 'image.png')),
+		)
+		assert.strictEqual(result.type, 'binary')
+	})
+
+	it('reports directories as not-file', async () => {
+		const result = await readTextFileForContext(
+			vscode.Uri.file(path.join(baseDir, 'folder')),
+		)
+		assert.strictEqual(result.type, 'not-file')
+	})
+})
+
 describe('file-system: abort handling (no workspace required)', () => {
 	it('getDirectoryChildren rejects when the signal is already aborted', async () => {
 		const controller = new AbortController()
@@ -129,6 +166,7 @@ describe('file-system: lazy tree APIs (workspace-scoped)', () => {
 	let restore: (() => void) | null = null
 
 	before(async function () {
+		clearIgnoreContextCache()
 		await fsp.rm(baseDir, { recursive: true, force: true })
 		await fsp.mkdir(baseDir, { recursive: true })
 		await writeFixture('src/app.ts', 'export const a = 1')
@@ -147,6 +185,7 @@ describe('file-system: lazy tree APIs (workspace-scoped)', () => {
 
 	after(async () => {
 		restore?.()
+		clearIgnoreContextCache()
 		await fsp.rm(baseDir, { recursive: true, force: true })
 	})
 
@@ -207,4 +246,18 @@ describe('file-system: lazy tree APIs (workspace-scoped)', () => {
 			'no gitignored files',
 		)
 	})
+
+	it('clearIgnoreContextCache lets callers pick up changed ignore rules', async () => {
+		await writeFixture('.gitignore', '')
+		clearIgnoreContextCache()
+
+		const parent = vscode.Uri.file(baseDir).toString()
+		const { roots: children } = await getDirectoryChildren(
+			parent,
+			DEFAULT_EXCLUDES,
+		)
+		const labels = children.map((c) => c.label)
+
+		assert.ok(labels.includes('ignored'), 'updated gitignore rules applied')
+	})
 })

src/utils/file-system.ts

@@ -126,6 +126,11 @@ export interface FileTreeResult {
 	truncated: boolean
 }
 
+export interface TextFileReadResult {
+	type: 'text' | 'binary' | 'not-file'
+	content?: string
+}
+
 interface IgnoreContext {
 	ign: ignore.Ignore
 	userIgnore: ignore.Ignore
@@ -137,6 +142,16 @@ interface ScanBudget {
 	truncated: boolean
 }
 
+const IGNORE_CONTEXT_CACHE_TTL_MS = 10_000
+const ignoreContextCache = new Map<
+	string,
+	{
+		expiresAt: number
+		context?: IgnoreContext
+		pending?: Promise<IgnoreContext>
+	}
+>()
+
 function throwIfAborted(signal?: AbortSignal): void {
 	if (signal?.aborted) {
 		const err = new Error('Aborted')
@@ -222,16 +237,21 @@ export function looksBinary(chunk: Uint8Array): boolean {
 	return analyzeByteContent(chunk)
 }
 
+export function hasBinaryFileExtension(uri: vscode.Uri): boolean {
+	const fileName = uri.path.toLowerCase()
+	const lastDot = fileName.lastIndexOf('.')
+	if (lastDot === -1) return false
+	return BINARY_EXTENSIONS.has(fileName.substring(lastDot))
+}
+
 export async function isBinaryFile(uri: vscode.Uri): Promise<boolean> {
 	try {
 		const stats = await vscode.workspace.fs.stat(uri)
 		if (stats.type !== vscode.FileType.File) {
 			return false
 		}
 
-		const fileName = uri.path.toLowerCase()
-		const extension = fileName.substring(fileName.lastIndexOf('.'))
-		if (BINARY_EXTENSIONS.has(extension)) {
+		if (hasBinaryFileExtension(uri)) {
 			return true
 		}
 
@@ -243,6 +263,30 @@ export async function isBinaryFile(uri: vscode.Uri): Promise<boolean> {
 	}
 }
 
+export async function readTextFileForContext(
+	uri: vscode.Uri,
+): Promise<TextFileReadResult> {
+	const stat = await vscode.workspace.fs.stat(uri)
+	if (stat.type !== vscode.FileType.File) {
+		return { type: 'not-file' }
+	}
+
+	if (hasBinaryFileExtension(uri)) {
+		return { type: 'binary' }
+	}
+
+	const contentBuffer = await vscode.workspace.fs.readFile(uri)
+	const chunk = contentBuffer.slice(0, 8000)
+	if (looksBinary(chunk)) {
+		return { type: 'binary' }
+	}
+
+	return {
+		type: 'text',
+		content: Buffer.from(contentBuffer).toString('utf8'),
+	}
+}
+
 function resolveExcludesPath(input: string): string {
 	let p = input.trim()
 	if (p.startsWith('~')) {
@@ -353,6 +397,22 @@ function buildIgnoreContext(
 	return { ign, userIgnore, rootUri }
 }
 
+export function clearIgnoreContextCache(): void {
+	ignoreContextCache.clear()
+}
+
+function getIgnoreContextCacheKey(
+	rootUri: vscode.Uri,
+	excludedDirs: string[],
+	useGitignore: boolean,
+): string {
+	return JSON.stringify({
+		root: rootUri.toString(),
+		useGitignore,
+		excludedDirs,
+	})
+}
+
 function findWorkspaceFolderForUri(
 	uri: vscode.Uri,
 ): vscode.WorkspaceFolder | undefined {
@@ -378,8 +438,38 @@ async function createIgnoreContextForUri(
 		throw new Error('No workspace folder for URI')
 	}
 	const useGitignore = options?.useGitignore !== false
-	const lines = await loadIgnoreRulesForRoot(folder.uri.fsPath, useGitignore)
-	return buildIgnoreContext(folder.uri, excludedDirs, lines)
+	const cacheKey = getIgnoreContextCacheKey(
+		folder.uri,
+		excludedDirs,
+		useGitignore,
+	)
+	const cached = ignoreContextCache.get(cacheKey)
+	if (cached?.context && cached.expiresAt > Date.now()) {
+		return cached.context
+	}
+	if (cached?.pending) {
+		return cached.pending
+	}
+
+	const pending = loadIgnoreRulesForRoot(folder.uri.fsPath, useGitignore)
+		.then((lines) => buildIgnoreContext(folder.uri, excludedDirs, lines))
+		.then((context) => {
+			ignoreContextCache.set(cacheKey, {
+				expiresAt: Date.now() + IGNORE_CONTEXT_CACHE_TTL_MS,
+				context,
+			})
+			return context
+		})
+		.catch((error) => {
+			ignoreContextCache.delete(cacheKey)
+			throw error
+		})
+
+	ignoreContextCache.set(cacheKey, {
+		expiresAt: 0,
+		pending,
+	})
+	return pending
 }
 
 function shouldIgnorePath(