fix(aws-serverless): Keep the Lambda extension polling past 300s invocations

`/event/next` acknowledges the previous event and waits for the next one, so the
poll stays open for the whole of the following invocation. Node's `fetch` caps
that at undici's 300s `headersTimeout`, and the rejection escapes a loop with no
`try`/`catch` — the extension never asks for another event, and Lambda holds
every later invocation on that execution environment until the function timeout.

The poll now uses `http.request`, which has no default timeout, with a deadline
above the 900s Lambda ceiling so a stalled socket still settles. A failed poll is
retried with capped backoff instead of ending the loop, except for a 4xx, which
never starts working. Registration fails when the Extensions API returns 200
without an identifier rather than leaving the id null.

`next` returns the event so the loop stops on SHUTDOWN, which the extension
registered for and ignored. Failures are reported through `console`: `debug` is
only enabled from `Sentry.init`, which this process never calls.

Fixes #24218

Co-authored-by: Claude Opus 5 <noreply@anthropic.com>

Author: LuccaRebelloToledo

packages/aws-serverless/src/lambda-extension/aws-lambda-extension.ts

@@ -10,6 +10,85 @@ import {
 } from '@sentry/core';
 import { DEBUG_BUILD } from './debug-build';
 
+const POLL_RETRY_BASE_MS = 100;
+const POLL_RETRY_MAX_MS = 5_000;
+const POLL_RETRY_MAX_EXPONENT = 6;
+
+/**
+ * Deliberately above the 900s Lambda ceiling: the poll must never be cut while an invocation is
+ * still running, but it must still settle if the connection goes stale without a FIN/RST. A poll
+ * that never settles parks the extension, and Lambda then holds every later invocation open until
+ * the function timeout kills it.
+ */
+export const POLL_TIMEOUT_MS = 960_000;
+
+interface ExtensionEvent {
+  eventType?: string;
+}
+
+interface ExtensionsApiResponse {
+  statusCode: number;
+  body: string;
+}
+
+export class ExtensionsApiError extends Error {
+  public constructor(
+    message: string,
+    public readonly statusCode: number,
+  ) {
+    super(message);
+    this.name = 'ExtensionsApiError';
+  }
+}
+
+/**
+ * Structural rather than `instanceof`: the check has to hold for an error that crossed a
+ * module boundary, and a transport failure carries `code`, never `statusCode`.
+ */
+function isClientError(err: unknown): boolean {
+  const statusCode = (err as { statusCode?: unknown } | null)?.statusCode;
+  return typeof statusCode === 'number' && statusCode >= 400 && statusCode < 500;
+}
+
+/**
+ * Exported only for testing purposes.
+ *
+ * `fetch` cannot be used for the long poll: Node's implementation applies undici's 300s
+ * `headersTimeout`, and lifting it would mean passing a dispatcher and depending on `undici`
+ * directly. `http.request` has no default timeout, and the Extensions API is plain HTTP on
+ * localhost.
+ */
+export function request(
+  url: string,
+  headers: Record<string, string>,
+  timeoutMs: number,
+): Promise<ExtensionsApiResponse> {
+  return new Promise((resolve, reject) => {
+    const req = http.request(url, { headers }, res => {
+      const chunks: Buffer[] = [];
+      res.on('data', (chunk: Buffer) => chunks.push(chunk));
+      res.on('end', () => resolve({ statusCode: res.statusCode ?? 0, body: Buffer.concat(chunks).toString() }));
+      res.on('error', err => {
+        req.destroy();
+        reject(err);
+      });
+    });
+
+    req.setTimeout(timeoutMs, () => {
+      req.destroy(new Error(`Extensions API did not respond within ${timeoutMs}ms`));
+    });
+
+    req.on('error', reject);
+    req.end();
+  });
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise(resolve => {
+    setTimeout(resolve, ms);
+  });
+}
+
 /**
  * The Extension API Client.
  */
@@ -42,25 +121,80 @@ export class AwsLambdaExtension {
     }
 
     this._extensionId = res.headers.get('lambda-extension-identifier');
+
+    if (!this._extensionId) {
+      throw new Error('Extensions API accepted the registration without returning an extension identifier');
+    }
   }
 
   /**
-   * Advances the extension to the next event.
+   * Advances the extension to the next event and returns it.
    */
-  public async next(): Promise<void> {
+  public async next(): Promise<ExtensionEvent> {
     if (!this._extensionId) {
       throw new Error('Extension ID is not set');
     }
 
-    const res = await fetch(`${this._baseUrl}/event/next`, {
-      headers: {
+    // This request blocks until the next event arrives, so it stays open for the whole
+    // duration of the current invocation. Under `fetch` that is capped at 300s, so any
+    // invocation that runs longer than that loses the extension partway through.
+    const res = await request(
+      `${this._baseUrl}/event/next`,
+      {
         'Lambda-Extension-Identifier': this._extensionId,
         'Content-Type': 'application/json',
       },
-    });
+      POLL_TIMEOUT_MS,
+    );
 
-    if (!res.ok) {
-      throw new Error(`Failed to advance to next event: ${await res.text()}`);
+    if (res.statusCode < 200 || res.statusCode > 299) {
+      throw new ExtensionsApiError(`Failed to advance to next event: ${res.body}`, res.statusCode);
+    }
+
+    try {
+      return JSON.parse(res.body) as ExtensionEvent;
+    } catch {
+      return {};
+    }
+  }
+
+  /**
+   * Polls the Extensions API until the environment shuts down.
+   *
+   * A failed poll is retried rather than ending the loop. Lambda only completes an invocation
+   * once the runtime and every registered extension have asked for the next event, so an
+   * extension that stops polling does not fail loudly — it leaves every later invocation on
+   * that execution environment running until the function timeout kills it.
+   */
+  public async run(): Promise<void> {
+    let consecutiveFailures = 0;
+
+    for (;;) {
+      try {
+        const event = await this.next();
+        consecutiveFailures = 0;
+
+        // The runtime API is torn down right after this, so polling again would only produce
+        // errors on the way out.
+        if (event.eventType === 'SHUTDOWN') {
+          return;
+        }
+      } catch (err) {
+        // A rejected registration is not going to start working; retrying only buries it.
+        if (isClientError(err)) {
+          throw err;
+        }
+
+        consecutiveFailures++;
+
+        consoleSandbox(() => {
+          // eslint-disable-next-line no-console
+          console.error('Sentry Lambda extension: polling the Extensions API failed, retrying.', err);
+        });
+
+        const exponent = Math.min(consecutiveFailures - 1, POLL_RETRY_MAX_EXPONENT);
+        await sleep(Math.min(POLL_RETRY_BASE_MS * 2 ** exponent, POLL_RETRY_MAX_MS));
+      }
     }
   }
 

packages/aws-serverless/src/lambda-extension/index.ts

@@ -1,7 +1,6 @@
 #!/usr/bin/env node
-import { debug } from '@sentry/core';
+import { consoleSandbox } from '@sentry/core';
 import { AwsLambdaExtension } from './aws-lambda-extension';
-import { DEBUG_BUILD } from './debug-build';
 
 async function main(): Promise<void> {
   const extension = new AwsLambdaExtension();
@@ -10,12 +9,14 @@ async function main(): Promise<void> {
 
   extension.startSentryTunnel();
 
-  // eslint-disable-next-line no-constant-condition
-  while (true) {
-    await extension.next();
-  }
+  await extension.run();
 }
 
 main().catch(err => {
-  DEBUG_BUILD && debug.error('Error in Lambda Extension', err);
+  // The debug logger is only enabled from `Sentry.init`, and this process never calls it, so
+  // nothing reported through the logger from here would ever be visible.
+  consoleSandbox(() => {
+    // eslint-disable-next-line no-console
+    console.error('Sentry Lambda extension: stopped, events will no longer be tunnelled.', err);
+  });
 });