diff --git a/.jules/bolt.md b/.jules/bolt.md new file mode 100644 index 00000000..fed302c9 --- /dev/null +++ b/.jules/bolt.md @@ -0,0 +1,3 @@ +## 2024-05-24 - Batch concurrent I/O in RejectionManager.list +**Learning:** Sequential await loops on disk I/O are a severe performance bottleneck for listing files. By implementing a standard batched concurrent map approach (Promise.all with a chunk size of 10), we can significantly improve listing performance while preventing EMFILE limits. +**Action:** Always batch file reading and modification across the codebase with chunked `Promise.all` when iterating over collections of files. \ No newline at end of file diff --git a/src/core/grizzco/execution/RejectionManager.ts b/src/core/grizzco/execution/RejectionManager.ts index 8bc0c717..270de697 100644 --- a/src/core/grizzco/execution/RejectionManager.ts +++ b/src/core/grizzco/execution/RejectionManager.ts @@ -50,23 +50,33 @@ export class RejectionManager { async list(): Promise { try { const files = await fs.readdir(this.rejectDir); + const rejFiles = files.filter((f) => f.endsWith('.rej')); const rejections: Rejection[] = []; - for (const file of files) { - if (!file.endsWith('.rej')) continue; + // Chunk size of 10 to prevent EMFILE limits while improving throughput + for (let i = 0; i < rejFiles.length; i += 10) { + const chunk = rejFiles.slice(i, i + 10); + const results = await Promise.all( + chunk.map(async (file) => { + try { + const content = await fs.readFile(path.join(this.rejectDir, file), 'utf-8'); + const headerPart = content.split('\n\n')[0]; + const header = JSON.parse(headerPart); + return { + filePath: file.replace('.rej', '').replace(/_/g, '/'), // Approximate restoration + ...header, + }; + } catch (error) { + getLogger().debug( + `[RejectionManager] Malformed rejection file ${file}: ${error instanceof Error ? error.message : String(error)}`, + ); + return null; + } + }), + ); - try { - const content = await fs.readFile(path.join(this.rejectDir, file), 'utf-8'); - const headerPart = content.split('\n\n')[0]; - const header = JSON.parse(headerPart); - rejections.push({ - filePath: file.replace('.rej', '').replace(/_/g, '/'), // Approximate restoration - ...header, - }); - } catch (error) { - getLogger().debug( - `[RejectionManager] Malformed rejection file ${file}: ${error instanceof Error ? error.message : String(error)}`, - ); + for (const res of results) { + if (res) rejections.push(res); } } diff --git a/tests/unit/core/grizzco/execution/executor.test.ts b/tests/unit/core/grizzco/execution/executor.test.ts index 6fbf9014..b41120d0 100644 --- a/tests/unit/core/grizzco/execution/executor.test.ts +++ b/tests/unit/core/grizzco/execution/executor.test.ts @@ -1,3 +1,4 @@ +import { describe, it, expect, mock } from 'bun:test'; import { WorkerFactory } from '../../../../../src/core/grizzco/execution/WorkerFactory.js'; import { createMockContext } from '../mocks.js';