Merge branch 'main' into docs-ai-agents-chat-agent-guide

Author: D-K-P

.server-changes/debounce-hidden-ceiling.md

@@ -0,0 +1,9 @@
+---
+area: webapp
+type: fix
+---
+
+Fix debounce doing nothing when the delay was longer than an hour, which made every trigger
+create its own run instead of collapsing onto the debounce key. Debounced runs now keep being
+pushed back for as long as triggers keep arriving, so set `maxDelay` when the work has to happen
+eventually, and settings that could never debounce are rejected rather than silently ignored.

apps/webapp/app/env.server.ts

@@ -114,6 +114,18 @@ const OptionalIntEnv = z.preprocess(
   z.coerce.number().int().optional()
 );
 
+/**
+ * Optional int env var for a limit that can be switched off. Blank, whitespace and `0` all mean
+ * "no limit" and normalise to undefined; anything else that is set must be greater than zero.
+ */
+const OptionalLimitEnv = z.preprocess((v) => {
+  if (typeof v === "string" && (v.trim() === "" || Number(v.trim()) === 0)) {
+    return undefined;
+  }
+
+  return v === 0 ? undefined : v;
+}, z.coerce.number().int().positive().optional());
+
 const EnvironmentSchema = z
   .object({
     NODE_ENV: z.union([z.literal("development"), z.literal("production"), z.literal("test")]),
@@ -1034,11 +1046,16 @@ const EnvironmentSchema = z
       .default(60_000),
     RUN_ENGINE_SUSPENDED_HEARTBEAT_RETRIES_FACTOR: z.coerce.number().default(2),
 
-    /** Maximum duration in milliseconds that a run can be debounced. Default: 1 hour (3,600,000ms) */
-    RUN_ENGINE_MAXIMUM_DEBOUNCE_DURATION_MS: z.coerce
-      .number()
-      .int()
-      .default(60_000 * 60), // 1 hour
+    /**
+     * Optional ceiling on how long a debounced run can be pushed back, measured from the first
+     * trigger. Unset by default: a continuously triggered debounce key is pushed back for as
+     * long as the triggers keep coming, and `debounce.maxDelay` on the trigger is the only
+     * bound. Setting this applies a ceiling to every debounced run that does not carry its own
+     * `maxDelay`, and any `delay` at or above it is rejected at trigger time. It is a default
+     * rather than an enforced limit: a trigger that sets `maxDelay` uses that value even when it
+     * is longer than this. `0` and blank both mean no ceiling.
+     */
+    RUN_ENGINE_MAXIMUM_DEBOUNCE_DURATION_MS: OptionalLimitEnv,
 
     /**
      * Bucket size in milliseconds used to quantize the newly computed `delayUntil`

apps/webapp/app/runEngine/services/triggerTask.server.ts

@@ -7,13 +7,15 @@ import type { Tracer } from "@opentelemetry/api";
 import { tryCatch } from "@trigger.dev/core/utils";
 import {
   type TriggerTaskRequestBody,
+  formatDurationMilliseconds,
   RunAnnotations,
   TaskRunError,
   taskRunErrorEnhancer,
   taskRunErrorToString,
   TriggerTraceContext,
 } from "@trigger.dev/core/v3";
 import {
+  parseNaturalLanguageDurationInMs,
   parseTraceparent,
   RunId,
   serializeTraceparent,
@@ -89,6 +91,7 @@ export class RunEngineTriggerTaskService {
   private readonly traceEventConcern: TraceEventConcern;
   private readonly triggerRacepointSystem: TriggerRacepointSystem;
   private readonly metadataMaximumSize: number;
+  private readonly maximumDebounceDurationMs: number | undefined;
   // Mollifier hooks are DI'd so tests can drive the call-site's mollify branch
   // deterministically (stub the gate to return mollify, inject a real or fake
   // buffer, force the global-enabled predicate to true so the call site
@@ -108,6 +111,7 @@ export class RunEngineTriggerTaskService {
     traceEventConcern: TraceEventConcern;
     tracer: Tracer;
     metadataMaximumSize: number;
+    maximumDebounceDurationMs?: number;
     triggerRacepointSystem?: TriggerRacepointSystem;
     evaluateGate?: MollifierEvaluateGate;
     getMollifierBuffer?: MollifierGetBuffer;
@@ -122,13 +126,74 @@ export class RunEngineTriggerTaskService {
     this.tracer = opts.tracer;
     this.traceEventConcern = opts.traceEventConcern;
     this.metadataMaximumSize = opts.metadataMaximumSize;
+    this.maximumDebounceDurationMs =
+      opts.maximumDebounceDurationMs ?? env.RUN_ENGINE_MAXIMUM_DEBOUNCE_DURATION_MS;
     this.triggerRacepointSystem = opts.triggerRacepointSystem ?? new NoopTriggerRacepointSystem();
     this.evaluateGate = opts.evaluateGate ?? defaultEvaluateGate;
     this.getMollifierBuffer = opts.getMollifierBuffer ?? defaultGetMollifierBuffer;
     this.isMollifierGloballyEnabled =
       opts.isMollifierGloballyEnabled ?? (() => env.TRIGGER_MOLLIFIER_ENABLED === "1");
   }
 
+  /**
+   * A debounced run is only pushed back while its new execution time stays inside the effective
+   * ceiling, which is the trigger's own `maxDelay` or, failing that, whatever ceiling the server
+   * is configured with. The room available to push is that ceiling minus `delay`, so a `delay`
+   * at or above it leaves none: the debounce key does nothing and every trigger creates its own
+   * run. Rejecting is better than accepting a trigger we know cannot debounce.
+   *
+   * With no `maxDelay` and no server ceiling there is nothing to conflict with, which is the
+   * default.
+   */
+  #validateDebounceWindow(
+    debounce: NonNullable<NonNullable<TriggerTaskRequestBody["options"]>["debounce"]>
+  ) {
+    const delayMs = parseNaturalLanguageDurationInMs(debounce.delay);
+
+    if (delayMs === undefined) {
+      throw new ServiceValidationError(
+        `Invalid debounce delay: ${debounce.delay}. debounce.delay must be a duration, not a ` +
+          `date, because it is re-applied every time the run is pushed back. Supported formats: ` +
+          `{number}s, {number}m, {number}h or {number}hr, {number}d, {number}w, optionally ` +
+          `combined (for example "2h30m").`
+      );
+    }
+
+    if (debounce.maxDelay !== undefined) {
+      const maxDelayMs = parseNaturalLanguageDurationInMs(debounce.maxDelay);
+
+      if (maxDelayMs === undefined) {
+        throw new ServiceValidationError(
+          `Invalid debounce maxDelay: ${debounce.maxDelay}. ` +
+            `Supported formats: {number}s, {number}m, {number}h or {number}hr, {number}d, ` +
+            `{number}w, optionally combined (for example "2h30m").`
+        );
+      }
+
+      if (maxDelayMs <= delayMs) {
+        throw new ServiceValidationError(
+          `debounce.maxDelay (${debounce.maxDelay}) must be longer than debounce.delay ` +
+            `(${debounce.delay}). A debounced run is only pushed back while it stays inside ` +
+            `maxDelay, so with these values every trigger would create its own run.`
+        );
+      }
+
+      return;
+    }
+
+    const serverCeilingMs = this.maximumDebounceDurationMs;
+
+    if (serverCeilingMs !== undefined && delayMs >= serverCeilingMs) {
+      throw new ServiceValidationError(
+        `debounce.delay (${debounce.delay}) is at or above this server's maximum debounce ` +
+          `duration of ${formatDurationMilliseconds(serverCeilingMs, { style: "short" })}. A ` +
+          `debounced run is only pushed back while it stays inside that window, so with this ` +
+          `delay every trigger would create its own run. Either shorten the delay, or set ` +
+          `debounce.maxDelay above ${debounce.delay}.`
+      );
+    }
+  }
+
   // Mint a new run's friendlyId. The id-kind decides which store the run is born
   // in (cuid → legacy store, run-ops id → new store), so the whole subgraph of a run
   // must agree. Two cases:
@@ -271,9 +336,12 @@ export class RunEngineTriggerTaskService {
             if (debounceDelayError || !debounceDelayUntil) {
               throw new ServiceValidationError(
                 `Invalid debounce delay: ${body.options.debounce.delay}. ` +
-                  `Supported formats: {number}s, {number}m, {number}h, {number}d, {number}w`
+                  `Supported formats: {number}s, {number}m, {number}h or {number}hr, {number}d, ` +
+                  `{number}w, optionally combined (for example "2h30m").`
               );
             }
+
+            this.#validateDebounceWindow(body.options.debounce);
           }
 
           const parentRun = body.options?.parentRunId

apps/webapp/test/engine/triggerTask.debounce.test.ts

@@ -459,4 +459,151 @@ describe("RunEngineTriggerTaskService", () => {
     expect(result).toBeDefined();
     expect(result?.run.friendlyId).toBeDefined();
   });
+
+  containerTest(
+    "should reject a debounce maxDelay that leaves no room to push the run back",
+    async ({ prisma, redisOptions }) => {
+      const engine = new RunEngine({
+        prisma,
+        worker: {
+          redis: redisOptions,
+          workers: 1,
+          tasksPerWorker: 10,
+          pollIntervalMs: 100,
+        },
+        queue: {
+          redis: redisOptions,
+        },
+        runLock: {
+          redis: redisOptions,
+        },
+        machines: {
+          defaultMachine: "small-1x",
+          machines: {
+            "small-1x": {
+              name: "small-1x" as const,
+              cpu: 0.5,
+              memory: 0.5,
+              centsPerMs: 0.0001,
+            },
+          },
+          baseCostInCents: 0.0005,
+        },
+        tracer: trace.getTracer("test", "0.0.0"),
+      });
+      onTestFinished(() => engine.quit());
+
+      const authenticatedEnvironment = await setupAuthenticatedEnvironment(prisma, "PRODUCTION");
+      const taskIdentifier = "test-task";
+
+      await setupBackgroundWorker(engine, authenticatedEnvironment, taskIdentifier);
+
+      const queuesManager = new DefaultQueueManager(prisma, engine);
+      const idempotencyKeyConcern = new IdempotencyKeyConcern(
+        prisma,
+        engine,
+        new MockTraceEventConcern()
+      );
+
+      const triggerTaskService = new RunEngineTriggerTaskService({
+        engine,
+        prisma,
+        payloadProcessor: new MockPayloadProcessor(),
+        queueConcern: queuesManager,
+        idempotencyKeyConcern,
+        validator: new MockTriggerTaskValidator(),
+        traceEventConcern: new MockTraceEventConcern(),
+        tracer: trace.getTracer("test", "0.0.0"),
+        metadataMaximumSize: 1024 * 1024 * 1,
+      });
+
+      const triggerWithDebounce = (debounce: { key: string; delay: string; maxDelay?: string }) =>
+        triggerTaskService.call({
+          taskId: taskIdentifier,
+          environment: authenticatedEnvironment,
+          body: { payload: { test: "test" }, options: { debounce } },
+        });
+
+      await expect(
+        triggerWithDebounce({ key: "equal", delay: "12h", maxDelay: "12h" })
+      ).rejects.toThrow(/must be longer than debounce.delay/);
+
+      await expect(
+        triggerWithDebounce({ key: "shorter", delay: "12h", maxDelay: "1h" })
+      ).rejects.toThrow(/must be longer than debounce.delay/);
+
+      await expect(
+        triggerWithDebounce({ key: "unparseable", delay: "10s", maxDelay: "soon" })
+      ).rejects.toThrow(/Invalid debounce maxDelay/);
+
+      await expect(
+        triggerWithDebounce({ key: "empty", delay: "10s", maxDelay: "" })
+      ).rejects.toThrow(/Invalid debounce maxDelay/);
+
+      const compound = await triggerWithDebounce({
+        key: "compound",
+        delay: "2h30m",
+        maxDelay: "1d",
+      });
+      expect(compound?.run.friendlyId).toBeDefined();
+
+      const withRoom = await triggerWithDebounce({
+        key: "with-room",
+        delay: "10s",
+        maxDelay: "5m",
+      });
+      expect(withRoom?.run.friendlyId).toBeDefined();
+
+      const noMaxDelay = await triggerWithDebounce({ key: "no-max-delay", delay: "12h" });
+      expect(noMaxDelay?.run.friendlyId).toBeDefined();
+
+      await expect(
+        triggerWithDebounce({ key: "date-delay", delay: "2027-01-01T00:00:00.000Z" })
+      ).rejects.toThrow(/must be a duration, not a date/);
+
+      const withServerCeiling = new RunEngineTriggerTaskService({
+        engine,
+        prisma,
+        payloadProcessor: new MockPayloadProcessor(),
+        queueConcern: queuesManager,
+        idempotencyKeyConcern,
+        validator: new MockTriggerTaskValidator(),
+        traceEventConcern: new MockTraceEventConcern(),
+        tracer: trace.getTracer("test", "0.0.0"),
+        metadataMaximumSize: 1024 * 1024 * 1,
+        maximumDebounceDurationMs: 60 * 60 * 1000,
+      });
+
+      await expect(
+        withServerCeiling.call({
+          taskId: taskIdentifier,
+          environment: authenticatedEnvironment,
+          body: {
+            payload: { test: "test" },
+            options: { debounce: { key: "over-server-ceiling", delay: "12h" } },
+          },
+        })
+      ).rejects.toThrow(/at or above this server's maximum debounce duration of 1h/);
+
+      const underServerCeiling = await withServerCeiling.call({
+        taskId: taskIdentifier,
+        environment: authenticatedEnvironment,
+        body: {
+          payload: { test: "test" },
+          options: { debounce: { key: "under-server-ceiling", delay: "10s" } },
+        },
+      });
+      expect(underServerCeiling?.run.friendlyId).toBeDefined();
+
+      const overCeilingWithMaxDelay = await withServerCeiling.call({
+        taskId: taskIdentifier,
+        environment: authenticatedEnvironment,
+        body: {
+          payload: { test: "test" },
+          options: { debounce: { key: "override", delay: "12h", maxDelay: "24h" } },
+        },
+      });
+      expect(overCeilingWithMaxDelay?.run.friendlyId).toBeDefined();
+    }
+  );
 });

docs/triggering.mdx

@@ -873,7 +873,7 @@ The `debounce` option accepts:
 - `key` - A unique string to identify the debounce group (scoped to the task)
 - `delay` - Duration string specifying how long to delay. Supported units: `s` (seconds), `m` (minutes), `h`/`hr` (hours), `d` (days), `w` (weeks). Minimum is 1 second. Examples: `"5s"`, `"1m"`, `"2h30m"`
 - `mode` - Optional. Controls which trigger's data is used: `"leading"` (default) or `"trailing"`
-- `maxDelay` - Optional. Maximum total time from the first trigger before the run must execute. Uses the same duration format as `delay`
+- `maxDelay` - Optional. Maximum total time from the first trigger before the run must execute. Uses the same duration format as `delay`. Not set by default
 
 **How it works:**
 
@@ -882,9 +882,18 @@ The `debounce` option accepts:
 3. Once no new triggers occur within the delay duration, the run executes
 4. After the run starts executing, a new trigger with the same key will create a new run
 
+<Warning>
+  There is no time limit on step 2. While triggers keep arriving on the same key, the run keeps
+  being pushed back and never executes. A key triggered every 10 seconds with a `delay` of `"30s"`
+  runs 30 seconds after the triggers stop, however long that takes. Set `maxDelay` whenever the
+  work needs to happen eventually. This matters most with `triggerAndWait`: every parent waiting
+  on a debounced run stays blocked, and holds its concurrency, for as long as the run keeps being
+  pushed back.
+</Warning>
+
 **Limiting total delay with `maxDelay`:**
 
-By default, continuous triggers can delay execution indefinitely. The `maxDelay` option sets an upper bound on the total delay from the first trigger, ensuring the run eventually executes even with constant activity.
+The `maxDelay` option sets an upper bound on the total delay from the first trigger, ensuring the run eventually executes even with constant activity.
 
 ```ts
 await summarizeChat.trigger(
@@ -921,11 +930,14 @@ Consider `delay: "5s"` and `maxDelay: "30s"` with triggers arriving every 2 seco
 
 Without `maxDelay`, continuous triggers would prevent the run from ever executing. With `maxDelay: "30s"`, execution is guaranteed within 30 seconds of the first trigger.
 
+Keep `delay` well below `maxDelay`. A run is only pushed back while its new execution time stays inside `maxDelay`, so the room you have to push is `maxDelay` minus `delay`. Setting them equal, or setting `delay` higher, leaves no room at all, and the trigger is rejected rather than accepted as a debounce that could never collapse anything.
+
 <Note>
   The `maxDelay` value is evaluated from each trigger call, not stored with the original run. This
   means if you pass different `maxDelay` values for the same debounce key, each trigger uses its own
-  `maxDelay` to check against the original run's creation time. For consistent behavior, use the
-  same `maxDelay` value for all triggers with the same debounce key.
+  `maxDelay` to check against the original run's creation time. A trigger that omits `maxDelay`
+  has no bound at all, so a single call without it can push the run past the limit the other calls
+  set. Use the same `maxDelay` value for every trigger with the same debounce key.
 </Note>
 
 **Leading vs Trailing mode:**

internal-packages/run-engine/src/engine/index.ts

@@ -384,7 +384,7 @@ export class RunEngine {
       redis: options.debounce?.redis ?? options.runLock.redis,
       executionSnapshotSystem: this.executionSnapshotSystem,
       delayedRunSystem: this.delayedRunSystem,
-      maxDebounceDurationMs: options.debounce?.maxDebounceDurationMs ?? 60 * 60 * 1000, // Default 1 hour
+      maxDebounceDurationMs: options.debounce?.maxDebounceDurationMs,
       quantizeNewDelayUntilMs: options.debounce?.quantizeNewDelayUntilMs ?? 1000,
       fastPathSkipEnabled: options.debounce?.fastPathSkipEnabled ?? true,
       useReplicaForFastPathRead: options.debounce?.useReplicaForFastPathRead ?? false,