From 03409c87f0b70749204048f0ef211f79e48b2e6e Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Mon, 2 Nov 2026 19:54:04 +0100 Subject: [PATCH] fix(report): one browser per bulk PDF report + capped time-limited render queue Each bulk report row called convertMarkupToPdf, which launched its own Chromium, and nothing capped how many browsers concurrent HTTP requests opened at once: a 59-row report spawned 59 browsers and Chromium gets OOM-killed under that load ("Session closed. Most likely the page has been closed." - 500 on GET /student/report with many ids). - BulkToPdfReportGenerator now opens one render scope around its row loop (node:async_hooks AsyncLocalStorage): every PDF conversion in the scope renders on a new page of ONE shared browser, the browser is closed once when the scope ends (also on the error path), and close() failures never mask the loop result or error. If the shared browser dies mid-report, the once-per-conversion retry relaunches it and the remaining rows continue on the fresh browser. Standalone conversions keep today's launch-per-conversion behavior. - Global browser cap with a FIFO wait queue: PDF_MAX_CONCURRENT_BROWSERS (default 2), PDF_QUEUE_WAIT_MS (default 20000). Expiring the wait throws a typed PdfRenderQueueTimeoutError instead of hanging until the reverse proxy kills the request. PDF_RENDER_BUDGET_MS (default 45000) is the per-conversion budget shared by all attempts: a retry gets only the remaining budget for its queue wait. Slots are released on every path (success, error, retry, queue timeout). - Each row also closes its page, so a many-row report cannot pile up open pages inside the shared browser. --- .../report/__tests__/pdf-render.queue.spec.ts | 174 ++++++++++++++ .../report/__tests__/pdf-render.scope.spec.ts | 160 +++++++++++++ utils/report/bulk-to-pdf.generator.ts | 24 +- utils/report/markup-to-pdf.generator.ts | 64 ++++-- utils/report/pdf-render.queue.ts | 212 ++++++++++++++++++ 5 files changed, 611 insertions(+), 23 deletions(-) create mode 100644 utils/report/__tests__/pdf-render.queue.spec.ts create mode 100644 utils/report/__tests__/pdf-render.scope.spec.ts create mode 100644 utils/report/pdf-render.queue.ts diff --git a/utils/report/__tests__/pdf-render.queue.spec.ts b/utils/report/__tests__/pdf-render.queue.spec.ts new file mode 100644 index 0000000..7d27aaf --- /dev/null +++ b/utils/report/__tests__/pdf-render.queue.spec.ts @@ -0,0 +1,174 @@ +import { MarkupToPdfReportGenerator } from '../markup-to-pdf.generator'; +import { PdfRenderQueueTimeoutError } from '../pdf-render.queue'; +import * as puppeteer from 'puppeteer'; +import { PDFDocument } from 'pdf-lib'; + +jest.mock('puppeteer'); + +describe('PDF render queue', () => { + class TestMarkupToPdfGenerator extends MarkupToPdfReportGenerator { + async getFileBuffer(data: any): Promise { + return this.convertMarkupToPdf(data); + } + } + + let generator: TestMarkupToPdfGenerator; + let mockBrowser: jest.Mocked; + let mockPage: jest.Mocked; + let onePagePdfBuffer: Buffer; + let pdfResolvers: Array<(pdf: Buffer) => void>; + let deferredPdf: () => Promise; + + beforeEach(async () => { + pdfResolvers = []; + deferredPdf = () => + new Promise((resolve) => { + pdfResolvers.push(resolve); + }); + + mockPage = { + setContent: jest.fn(), + pdf: jest.fn(), + on: jest.fn(), + setRequestInterception: jest.fn().mockResolvedValue(undefined), + close: jest.fn().mockResolvedValue(undefined), + } as any as jest.Mocked; + + mockBrowser = { + newPage: jest.fn().mockResolvedValue(mockPage), + close: jest.fn(), + } as any as jest.Mocked; + + (puppeteer.launch as jest.Mock).mockResolvedValue(mockBrowser); + + generator = new TestMarkupToPdfGenerator(() => 'test'); + + const onePagePdfDoc = await PDFDocument.create(); + onePagePdfDoc.addPage(); + onePagePdfBuffer = Buffer.from(await onePagePdfDoc.save()); + }); + + afterEach(() => { + jest.useRealTimers(); + delete process.env.PDF_MAX_CONCURRENT_BROWSERS; + delete process.env.PDF_QUEUE_WAIT_MS; + delete process.env.PDF_RENDER_BUDGET_MS; + jest.clearAllMocks(); + }); + + it('should queue a conversion above the cap and reject it with PdfRenderQueueTimeoutError after the configured wait', async () => { + process.env.PDF_MAX_CONCURRENT_BROWSERS = '2'; + process.env.PDF_QUEUE_WAIT_MS = '20000'; + jest.useFakeTimers(); + + mockPage.pdf.mockImplementation(deferredPdf); + const first = generator.convertMarkupToPdf('
first
'); + const second = generator.convertMarkupToPdf('
second
'); + + const third = generator.convertMarkupToPdf('
third
'); + + let rejection: any; + third.then( + () => { + rejection = new Error('third conversion should not resolve'); + }, + (e) => { + rejection = e; + }, + ); + await jest.advanceTimersByTimeAsync(20000); + + expect(rejection).toBeInstanceOf(PdfRenderQueueTimeoutError); + expect(rejection.message).toBe('PDF render queue wait exceeded 20000ms (max 2 concurrent browsers)'); + expect(puppeteer.launch).toHaveBeenCalledTimes(2); + + // No slot leak: once the holding conversions finish, the next conversion + // goes through instead of waiting forever. + pdfResolvers.forEach((resolve) => resolve(onePagePdfBuffer)); + await Promise.all([first, second]); + + const fourth = generator.convertMarkupToPdf('
fourth
'); + while (pdfResolvers.length < 3) { + await Promise.resolve(); + } + pdfResolvers[pdfResolvers.length - 1](onePagePdfBuffer); + await expect(fourth).resolves.toEqual(onePagePdfBuffer); + expect(puppeteer.launch).toHaveBeenCalledTimes(3); + }); + + it('should cap the retry queue wait by the remaining render budget', async () => { + process.env.PDF_MAX_CONCURRENT_BROWSERS = '2'; + process.env.PDF_QUEUE_WAIT_MS = '30000'; + process.env.PDF_RENDER_BUDGET_MS = '20000'; + jest.useFakeTimers(); + + const transientError = new Error( + 'Protocol error (Runtime.callFunctionOn): Session closed. Most likely the page has been closed.', + ); + + // Deterministic by markup: the conversion under test burns 5000ms of its + // budget with a stuck setContent; the two holder conversions render + // instantly and hold their slots with deferred pdf promises. + mockPage.setContent.mockImplementation((markup: string) => { + if (String(markup).includes('burn')) { + return new Promise((resolve) => setTimeout(resolve, 5000)); + } + return Promise.resolve(); + }); + mockPage.pdf + .mockImplementationOnce(deferredPdf) // holder conversion holds its slot + .mockImplementationOnce(() => Promise.reject(transientError)) // first (burning) attempt of `conversion` + .mockImplementation(deferredPdf); // all other conversions (incl. the second holder) + + const holder = generator.convertMarkupToPdf('
holder
'); + const conversion = generator.convertMarkupToPdf('
burn
'); + const secondHolder = generator.convertMarkupToPdf('
second holder
'); + + let rejection: any; + conversion.then( + () => { + rejection = new Error('conversion should not resolve'); + }, + (e) => { + rejection = e; + }, + ); + + // 5000ms burnt + 14999ms of the retry's queue wait: still inside budget. + await jest.advanceTimersByTimeAsync(5000 + 14999); + expect(rejection).toBeUndefined(); + + // The retry's wait runs out at exactly the remaining budget (20000 - 5000). + await jest.advanceTimersByTimeAsync(1000); + expect(rejection).toBeInstanceOf(PdfRenderQueueTimeoutError); + expect(rejection.message).toBe('PDF render queue wait exceeded 15000ms (max 2 concurrent browsers)'); + expect(puppeteer.launch).toHaveBeenCalledTimes(3); + + // No slot leak after the queue timeout: settling the holders frees the + // slots and the next conversion still gets through. + pdfResolvers.forEach((resolve) => resolve(onePagePdfBuffer)); + await Promise.all([holder, secondHolder]); + + const fourth = generator.convertMarkupToPdf('
after timeout
'); + while (pdfResolvers.length < 3) { + await Promise.resolve(); + } + pdfResolvers[pdfResolvers.length - 1](onePagePdfBuffer); + await expect(fourth).resolves.toEqual(onePagePdfBuffer); + expect(puppeteer.launch).toHaveBeenCalledTimes(4); + }); + + it('should clean up and let the next conversion through when everything settles', async () => { + // (tail of the two tests above; this one checks the simple failure path) + mockPage.pdf.mockRejectedValueOnce(new Error('pdf generation error')).mockResolvedValue(onePagePdfBuffer); + + await expect(generator.convertMarkupToPdf('
a
')).rejects.toThrow('pdf generation error'); + expect(mockBrowser.close).toHaveBeenCalledTimes(1); + + const result = await generator.convertMarkupToPdf('
b
'); + + expect(result).toEqual(onePagePdfBuffer); + expect(puppeteer.launch).toHaveBeenCalledTimes(2); + expect(mockBrowser.close).toHaveBeenCalledTimes(2); + }); +}); \ No newline at end of file diff --git a/utils/report/__tests__/pdf-render.scope.spec.ts b/utils/report/__tests__/pdf-render.scope.spec.ts new file mode 100644 index 0000000..e015ec2 --- /dev/null +++ b/utils/report/__tests__/pdf-render.scope.spec.ts @@ -0,0 +1,160 @@ +import { BulkToPdfReportGenerator } from '../bulk-to-pdf.generator'; +import { MarkupToPdfReportGenerator } from '../markup-to-pdf.generator'; +import * as puppeteer from 'puppeteer'; +import { PDFDocument } from 'pdf-lib'; + +jest.mock('puppeteer'); + +describe('BulkToPdfReportGenerator render scope', () => { + class TestMarkupToPdfGenerator extends MarkupToPdfReportGenerator { + async getFileBuffer(data: any): Promise { + return this.convertMarkupToPdf('
row
'); + } + } + + let mockBrowser: jest.Mocked; + let mockPage: jest.Mocked; + let onePagePdfBuffer: Buffer; + + beforeEach(async () => { + mockPage = { + setContent: jest.fn(), + pdf: jest.fn(), + on: jest.fn(), + setRequestInterception: jest.fn().mockResolvedValue(undefined), + close: jest.fn().mockResolvedValue(undefined), + } as any as jest.Mocked; + + mockBrowser = { + newPage: jest.fn().mockResolvedValue(mockPage), + close: jest.fn(), + } as any as jest.Mocked; + + (puppeteer.launch as jest.Mock).mockResolvedValue(mockBrowser); + + const onePagePdfDoc = await PDFDocument.create(); + onePagePdfDoc.addPage(); + onePagePdfBuffer = Buffer.from(await onePagePdfDoc.save()); + }); + + afterEach(() => { + jest.useRealTimers(); + delete process.env.PDF_MAX_CONCURRENT_BROWSERS; + delete process.env.PDF_QUEUE_WAIT_MS; + jest.clearAllMocks(); + }); + + it('should launch exactly one browser for a 3 row report and close it once', async () => { + mockPage.pdf.mockResolvedValue(onePagePdfBuffer); + const bulkGenerator = new BulkToPdfReportGenerator(new TestMarkupToPdfGenerator(() => 'test')); + + const result = await bulkGenerator.getFileBuffer([1, 2, 3]); + + expect(puppeteer.launch).toHaveBeenCalledTimes(1); + expect(puppeteer.launch).toHaveBeenCalledWith({ + args: ['--disable-dev-shm-usage', '--no-sandbox', '--disable-setuid-sandbox'], + }); + expect(mockBrowser.newPage).toHaveBeenCalledTimes(3); + expect(mockPage.close).toHaveBeenCalledTimes(3); + expect(mockBrowser.close).toHaveBeenCalledTimes(1); + expect((await PDFDocument.load(result)).getPageCount()).toBe(3); + }); + + it('should close the shared browser once when a row fails', async () => { + mockPage.pdf + .mockResolvedValueOnce(onePagePdfBuffer) + .mockRejectedValueOnce(new Error('row 2 pdf generation error')); + const bulkGenerator = new BulkToPdfReportGenerator(new TestMarkupToPdfGenerator(() => 'test')); + + await expect(bulkGenerator.getFileBuffer([1, 2, 3])).rejects.toThrow('row 2 pdf generation error'); + + expect(puppeteer.launch).toHaveBeenCalledTimes(1); + expect(mockPage.close).toHaveBeenCalledTimes(2); + expect(mockBrowser.close).toHaveBeenCalledTimes(1); + }); + + it('should relaunch the shared browser at most once for a transient failure and continue remaining rows', async () => { + const transientError = new Error( + 'Protocol error (Runtime.callFunctionOn): Session closed. Most likely the page has been closed.', + ); + mockPage.pdf + .mockRejectedValueOnce(transientError) + .mockResolvedValue(onePagePdfBuffer); + const bulkGenerator = new BulkToPdfReportGenerator(new TestMarkupToPdfGenerator(() => 'test')); + + const result = await bulkGenerator.getFileBuffer([1, 2, 3]); + + expect(puppeteer.launch).toHaveBeenCalledTimes(2); + expect(mockBrowser.newPage).toHaveBeenCalledTimes(4); + expect(await PDFDocument.load(result)).not.toBeNull(); + expect((await PDFDocument.load(result)).getPageCount()).toBe(3); + }); + + it('should not leak a render slot when the report relaunches its browser', async () => { + process.env.PDF_MAX_CONCURRENT_BROWSERS = '1'; + process.env.PDF_QUEUE_WAIT_MS = '1000'; + jest.useFakeTimers(); + + const transientError = new Error( + 'Protocol error (Runtime.callFunctionOn): Session closed. Most likely the page has been closed.', + ); + mockPage.pdf + .mockRejectedValueOnce(transientError) + .mockResolvedValue(onePagePdfBuffer); + const bulkGenerator = new BulkToPdfReportGenerator(new TestMarkupToPdfGenerator(() => 'test')); + + const result = await bulkGenerator.getFileBuffer([1, 2, 3]); + + expect(puppeteer.launch).toHaveBeenCalledTimes(2); // shared browser relaunched once + + // With a cap of 1, a leaked slot would make this conversion wait in the + // queue and reject instead of rendering immediately. + const next = new TestMarkupToPdfGenerator(() => 'test').convertMarkupToPdf('
next
'); + await expect(next).resolves.toEqual(onePagePdfBuffer); + expect(puppeteer.launch).toHaveBeenCalledTimes(3); + }); +}); + +describe('MarkupToPdfReportGenerator outside a scope', () => { + // Sanity check: without a bulk report around it, every conversion still owns + // its own browser. + class TestMarkupToPdfGenerator extends MarkupToPdfReportGenerator { + async getFileBuffer(data: any): Promise { + return this.convertMarkupToPdf('
row
'); + } + } + + let mockBrowser: jest.Mocked; + let mockPage: jest.Mocked; + + beforeEach(() => { + mockPage = { + setContent: jest.fn(), + pdf: jest.fn().mockResolvedValue(Buffer.from('pdf content')), + on: jest.fn(), + setRequestInterception: jest.fn().mockResolvedValue(undefined), + close: jest.fn().mockResolvedValue(undefined), + } as any as jest.Mocked; + + mockBrowser = { + newPage: jest.fn().mockResolvedValue(mockPage), + close: jest.fn(), + } as any as jest.Mocked; + + (puppeteer.launch as jest.Mock).mockResolvedValue(mockBrowser); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should launch and close a browser per standalone conversion', async () => { + const generator = new TestMarkupToPdfGenerator(() => 'test'); + + await generator.convertMarkupToPdf('
a
'); + await generator.convertMarkupToPdf('
b
'); + + expect(puppeteer.launch).toHaveBeenCalledTimes(2); + expect(mockBrowser.close).toHaveBeenCalledTimes(2); + }); +}); \ No newline at end of file diff --git a/utils/report/bulk-to-pdf.generator.ts b/utils/report/bulk-to-pdf.generator.ts index f18883f..e20269e 100644 --- a/utils/report/bulk-to-pdf.generator.ts +++ b/utils/report/bulk-to-pdf.generator.ts @@ -2,6 +2,7 @@ import { DataSource } from "typeorm"; import { PDFDocument } from 'pdf-lib'; import { BaseReportGenerator } from "./report.generators"; import { CommonFileFormat } from "./types"; +import { runPdfRenderScope } from "./pdf-render.queue"; export class BulkToPdfReportGenerator extends BaseReportGenerator { fileFormat: CommonFileFormat = CommonFileFormat.Pdf; @@ -19,15 +20,20 @@ export class BulkToPdfReportGenerator extends BaseReportGenerator { } async getFileBuffer(data: any[]): Promise { - const pdfDoc = await PDFDocument.create() - for (const item of data) { - const ItemPdfBytes = await this.generator.getFileBuffer(item); - const itemPdfDoc = await PDFDocument.load(ItemPdfBytes); - const itemPdfPages = await pdfDoc.copyPages(itemPdfDoc, itemPdfDoc.getPageIndices()); - itemPdfPages.forEach((page) => pdfDoc.addPage(page)); - } - const pdfBytes = await pdfDoc.save() + // One render scope for the whole report: rows render on pages of ONE + // shared browser (opened by the first PDF conversion in the scope) and + // it is closed once when the scope ends - success or throw. + return runPdfRenderScope(async () => { + const pdfDoc = await PDFDocument.create() + for (const item of data) { + const ItemPdfBytes = await this.generator.getFileBuffer(item); + const itemPdfDoc = await PDFDocument.load(ItemPdfBytes); + const itemPdfPages = await pdfDoc.copyPages(itemPdfDoc, itemPdfDoc.getPageIndices()); + itemPdfPages.forEach((page) => pdfDoc.addPage(page)); + } + const pdfBytes = await pdfDoc.save() - return Buffer.from(pdfBytes); + return Buffer.from(pdfBytes); + }); } } diff --git a/utils/report/markup-to-pdf.generator.ts b/utils/report/markup-to-pdf.generator.ts index f85797e..b2311db 100644 --- a/utils/report/markup-to-pdf.generator.ts +++ b/utils/report/markup-to-pdf.generator.ts @@ -1,9 +1,14 @@ import puppeteer from 'puppeteer'; -import type { Page } from 'puppeteer'; +import type { Browser, Page } from 'puppeteer'; import { BaseReportGenerator } from './report.generators'; import { CommonFileFormat } from './types'; - -const PUPPETEER_LAUNCH_ARGS = ['--disable-dev-shm-usage', '--no-sandbox', '--disable-setuid-sandbox']; +import { + acquireBrowserSlot, + PUPPETEER_LAUNCH_ARGS, + getPdfRenderBudgetMs, + getPdfRenderScope, + releaseBrowserSlot, +} from './pdf-render.queue'; // Browser processes can die mid-render under load (e.g. killed by the OOM killer // while several reports render at once), surfacing as protocol errors like @@ -48,22 +53,54 @@ export abstract class MarkupToPdfReportGenerator extends BaseR return Buffer.from(markup); } + // One render budget per convertMarkupToPdf call: the queue wait of a + // retry is capped by the remaining budget, never by a fresh full wait. + const queueDeadline = Date.now() + getPdfRenderBudgetMs(); + try { - return await this.generatePdfInFreshBrowser(markup); + return await this.renderPdfAttempt(markup, queueDeadline, false); } catch (e) { if (!isTransientPuppeteerError(e)) { throw e; } - return await this.generatePdfInFreshBrowser(markup); + return await this.renderPdfAttempt(markup, queueDeadline, true); } } - private async generatePdfInFreshBrowser(markup: string): Promise { + // One attempt: either render on the browser of the active report render + // scope (one browser per bulk report), or on a browser of this conversion's + // own, launched inside a globally capped render-queue slot. + private async renderPdfAttempt(markup: string, queueDeadline: number, isRetry: boolean): Promise { + const scope = getPdfRenderScope(); + if (scope) { + // Inside a render scope the report owns one browser: this conversion + // only opens its page on it and must not close the browser. A retry + // relaunches the shared browser at most once (restartBrowser). + const browser = isRetry ? await scope.restartBrowser(queueDeadline) : await scope.getBrowser(queueDeadline); + return this.renderPdfOnBrowser(browser, markup); + } + + await acquireBrowserSlot(queueDeadline); let browser; try { browser = await puppeteer.launch({ args: PUPPETEER_LAUNCH_ARGS }); - const page = await browser.newPage(); + return await this.renderPdfOnBrowser(browser, markup); + } finally { + // The browser may already be dead (crashed / OOM-killed), in which case + // close() itself can reject - don't let that mask the real result. + if (browser) { + try { + await browser.close(); + } catch {} + } + releaseBrowserSlot(); + } + } + + private async renderPdfOnBrowser(browser: Browser, markup: string): Promise { + const page = await browser.newPage(); + try { try { await page.setContent(markup, { waitUntil: 'networkidle0', timeout: 10000 }); } catch (e) { @@ -81,13 +118,12 @@ export abstract class MarkupToPdfReportGenerator extends BaseR timeout: getPdfPrintTimeoutMs(), }); } finally { - // The browser may already be dead (crashed / OOM-killed), in which case - // close() itself can reject - don't let that mask the real result. - if (browser) { - try { - await browser.close(); - } catch {} - } + // Rows of a shared scope browser take their page with them, so a + // many-row report can't pile up open pages; standalone browsers would + // close it together with the browser anyway. + try { + await page.close(); + } catch {} } } } diff --git a/utils/report/pdf-render.queue.ts b/utils/report/pdf-render.queue.ts new file mode 100644 index 0000000..16b0c3d --- /dev/null +++ b/utils/report/pdf-render.queue.ts @@ -0,0 +1,212 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; +import puppeteer from 'puppeteer'; +import type { Browser } from 'puppeteer'; + +// Every Chromium opened for PDF rendering (standalone conversion or shared +// render scope) is launched with the same args. +export const PUPPETEER_LAUNCH_ARGS = ['--disable-dev-shm-usage', '--no-sandbox', '--disable-setuid-sandbox']; + +// Hard cap on Chromium processes alive at once in this process. Browsers +// killed by the OOM killer under load are the direct cause of the +// "Session closed. Most likely the page has been closed." failures, so more +// concurrency is never better. PDF_MAX_CONCURRENT_BROWSERS overrides it. +function getMaxConcurrentBrowsers(): number { + const configured = Number(process.env.PDF_MAX_CONCURRENT_BROWSERS); + return Number.isFinite(configured) && configured > 0 ? configured : 2; +} + +// How long a conversion may wait in the render queue once the cap is full. +// Waiting until the reverse proxy kills the request is not acceptable: when +// the wait expires a PdfRenderQueueTimeoutError is thrown so callers can +// answer a clean 503. PDF_QUEUE_WAIT_MS overrides it. +function getQueueWaitMs(): number { + const configured = Number(process.env.PDF_QUEUE_WAIT_MS); + return Number.isFinite(configured) && configured > 0 ? configured : 20000; +} + +// Total wall-clock budget of one convertMarkupToPdf call, including the queue +// wait of each of its attempts. A retry is only granted the remaining budget +// for its queue wait, never a fresh full wait. PDF_RENDER_BUDGET_MS +// overrides it. +export function getPdfRenderBudgetMs(): number { + const configured = Number(process.env.PDF_RENDER_BUDGET_MS); + return Number.isFinite(configured) && configured > 0 ? configured : 45000; +} + +export class PdfRenderQueueTimeoutError extends Error { + constructor(allowedWaitMs: number, maxConcurrentBrowsers: number) { + super(`PDF render queue wait exceeded ${allowedWaitMs}ms (max ${maxConcurrentBrowsers} concurrent browsers)`); + this.name = 'PdfRenderQueueTimeoutError'; + } +} + +// ---------------------------------------------------------------- queue ---- + +let activeBrowserSlots = 0; + +interface BrowserSlotWaiter { + done: boolean; + timer: ReturnType; + grant: () => void; +} + +const browserSlotWaiters: BrowserSlotWaiter[] = []; + +// Blocks until one of the PDF_MAX_CONCURRENT_BROWSERS slots is free. Waits are +// FIFO and bounded by the smaller of PDF_QUEUE_WAIT_MS and the remaining time +// of the calling request's render budget; expiring gives a +// PdfRenderQueueTimeoutError and never launches a browser. +export async function acquireBrowserSlot(queueDeadline: number): Promise { + const maxConcurrentBrowsers = getMaxConcurrentBrowsers(); + if (activeBrowserSlots < maxConcurrentBrowsers) { + activeBrowserSlots++; + return; + } + + const allowedWaitMs = Math.min(getQueueWaitMs(), Math.max(0, queueDeadline - Date.now())); + if (allowedWaitMs <= 0) { + throw new PdfRenderQueueTimeoutError(0, maxConcurrentBrowsers); + } + + await new Promise((resolve, reject) => { + let waiter: BrowserSlotWaiter; + const timer = setTimeout(() => { + // Timed out: leave the queue without ever getting a slot, so a later + // release can't grant one to a dead waiter (that would leak it). + if (waiter.done) { + return; + } + const index = browserSlotWaiters.indexOf(waiter); + if (index !== -1) { + browserSlotWaiters.splice(index, 1); + } + waiter.done = true; + reject(new PdfRenderQueueTimeoutError(allowedWaitMs, maxConcurrentBrowsers)); + }, allowedWaitMs); + waiter = { + done: false, + timer, + grant: () => { + waiter.done = true; + clearTimeout(timer); + const index = browserSlotWaiters.indexOf(waiter); + if (index !== -1) { + browserSlotWaiters.splice(index, 1); + } + activeBrowserSlots++; + resolve(); + }, + }; + browserSlotWaiters.push(waiter); + }); +} + +// Frees the slot granted by acquireBrowserSlot and hands it to the +// longest-waiting live waiter. Must be called exactly once for every granted +// acquire, on every path (success, thrown error, retry); a leaked slot would +// deadlock all PDF rendering in this process. +export function releaseBrowserSlot(): void { + if (activeBrowserSlots > 0) { + activeBrowserSlots--; + } + while (browserSlotWaiters.length > 0) { + const waiter = browserSlotWaiters.shift(); + if (!waiter.done) { + waiter.grant(); + return; + } + } +} + +// ------------------------------------------------------------- scope ---- + +// One render scope per bulk report: a single browser owned by the report, +// reused via a new page for every row, closed once when the scope ends (also +// on the error path). A retry relaunches it at most once and the remaining +// rows continue on the fresh browser - no orphan browsers left behind. +export interface PdfRenderScope { + getBrowser(queueDeadline: number): Promise; + restartBrowser(queueDeadline: number): Promise; + closeBrowsers(): Promise; +} + +class PdfRenderScopeImpl implements PdfRenderScope { + private browser: Browser = null; + private slotHeld = false; + + async getBrowser(queueDeadline: number): Promise { + if (this.browser) { + return this.browser; + } + return this.launchAndHold(queueDeadline); + } + + async restartBrowser(queueDeadline: number): Promise { + const deadBrowser = this.browser; + this.browser = null; + if (deadBrowser) { + // The dead browser's close() can itself reject - the retry must not be + // masked by it. The report keeps its slot: relaunching only replaces the + // process behind it. + try { + await deadBrowser.close(); + } catch {} + } + return this.launchAndHold(queueDeadline); + } + + async closeBrowsers(): Promise { + const browser = this.browser; + this.browser = null; + if (browser) { + // Same as everywhere else: a dying browser's close() may reject and must + // not mask the report result or the report error. + try { + await browser.close(); + } catch {} + } + if (this.slotHeld) { + this.slotHeld = false; + releaseBrowserSlot(); + } + } + + private async launchAndHold(queueDeadline: number): Promise { + // A relaunch (restartBrowser) replaces the process behind an already-held + // slot and must NOT take a second one - that would leak a slot and slowly + // deadlock all PDF rendering. + const acquiredSlot = !this.slotHeld; + if (acquiredSlot) { + await acquireBrowserSlot(queueDeadline); + this.slotHeld = true; + } + try { + this.browser = await puppeteer.launch({ args: PUPPETEER_LAUNCH_ARGS }); + } catch (e) { + if (acquiredSlot) { + this.slotHeld = false; + releaseBrowserSlot(); + } + throw e; + } + return this.browser; + } +} + +const pdfRenderScopeStorage = new AsyncLocalStorage(); + +export function getPdfRenderScope(): PdfRenderScope | null { + return pdfRenderScopeStorage.getStore() ?? null; +} + +// Opens a render scope and guarantees its single browser is closed once the +// loop inside finishes or throws, without letting a failing close() mask the +// loop result or the loop error. +export async function runPdfRenderScope(render: () => Promise): Promise { + const scope = new PdfRenderScopeImpl(); + try { + return await pdfRenderScopeStorage.run(scope, render); + } finally { + await scope.closeBrowsers(); + } +} \ No newline at end of file