diff --git a/.jules/bolt.md b/.jules/bolt.md new file mode 100644 index 00000000..7416aed4 --- /dev/null +++ b/.jules/bolt.md @@ -0,0 +1,3 @@ +## 2025-02-24 - Batched Concurrent I/O for Large File Iterations +**Learning:** Sequential I/O operations (like `fs.stat` and `fs.rm`) when iterating over large numbers of files (e.g., in `ArtifactStore.gc`) create severe performance bottlenecks and increase execution time linearly. +**Action:** Always use batched concurrent I/O (e.g., `Promise.all` with chunk sizes of 10) for directories with many files to speed up execution and avoid EMFILE limits. Instantiate Promises only within the `Promise.all()` call for the specific chunk to avoid immediate concurrent execution upfront. diff --git a/src/core/sub-agent/artifacts/store.ts b/src/core/sub-agent/artifacts/store.ts index 8a7aff5c..2c295a7b 100644 --- a/src/core/sub-agent/artifacts/store.ts +++ b/src/core/sub-agent/artifacts/store.ts @@ -93,13 +93,29 @@ export class ArtifactStore { const entries = await fs.readdir(root, { withFileTypes: true }).catch(() => []); const files: Array<{ name: string; path: string; mtimeMs: number; size: number }> = []; - for (const entry of entries) { - if (!entry.isFile()) continue; + const validEntries = entries.filter((entry) => { + if (!entry.isFile()) return false; const filePath = path.join(root, entry.name); - if (!isWithinDir(root, filePath)) continue; - const stat = await fs.stat(filePath).catch(() => null); - if (!stat) continue; - files.push({ name: entry.name, path: filePath, mtimeMs: stat.mtimeMs, size: stat.size }); + return isWithinDir(root, filePath); + }); + + // ⚡ Bolt Optimization: Use batched concurrent fs.stat calls (chunk size 10) + // to significantly speed up file metadata resolution for large directories + // while preventing EMFILE (too many open files) limits. + for (let i = 0; i < validEntries.length; i += 10) { + const chunk = validEntries.slice(i, i + 10); + const stats = await Promise.all( + chunk.map(async (entry) => { + const filePath = path.join(root, entry.name); + const stat = await fs.stat(filePath).catch(() => null); + return { entry, filePath, stat }; + }), + ); + for (const { entry, filePath, stat } of stats) { + if (stat) { + files.push({ name: entry.name, path: filePath, mtimeMs: stat.mtimeMs, size: stat.size }); + } + } } const maxAgeMs = options?.maxAgeMs ?? LIMITS.artifactTtlMs; @@ -112,14 +128,19 @@ export class ArtifactStore { let removedFiles = 0; let removedBytes = 0; - const removeFile = async (file: { path: string; size: number }) => { - await fs.rm(file.path, { force: true }).catch(() => null); - removedFiles += 1; - removedBytes += file.size; - }; - - for (const file of expired) { - await removeFile(file); + // ⚡ Bolt Optimization: Delete expired files concurrently in batches of 10 + // to improve IO throughput instead of sequential awaits. + for (let i = 0; i < expired.length; i += 10) { + const chunk = expired.slice(i, i + 10); + await Promise.all( + chunk.map(async (file) => { + await fs.rm(file.path, { force: true }).catch(() => null); + }), + ); + for (const file of chunk) { + removedFiles += 1; + removedBytes += file.size; + } } // Recompute remaining after TTL removal (newest first). @@ -129,6 +150,7 @@ export class ArtifactStore { let currentFiles = remaining.length; let currentBytes = remaining.reduce((acc, f) => acc + f.size, 0); + const toRemove: Array<{ path: string; size: number }> = []; for (let i = remaining.length - 1; i >= 0; i--) { const tooManyFiles = currentFiles > maxFiles; @@ -136,11 +158,25 @@ export class ArtifactStore { if (!tooManyFiles && !tooManyBytes) break; const oldest = remaining[i]; - await removeFile(oldest); + toRemove.push(oldest); currentFiles -= 1; currentBytes -= oldest.size; } + // ⚡ Bolt Optimization: Delete capacity-exceeding files concurrently in batches of 10 + for (let i = 0; i < toRemove.length; i += 10) { + const chunk = toRemove.slice(i, i + 10); + await Promise.all( + chunk.map(async (file) => { + await fs.rm(file.path, { force: true }).catch(() => null); + }), + ); + for (const file of chunk) { + removedFiles += 1; + removedBytes += file.size; + } + } + return { removedFiles, removedBytes }; }