fix(redis): stop a slow handshake from failing as a command timeout

ioredis arms the commandTimeout timer in sendCommand before it checks
whether the socket is writable, so the budget covers connection setup and
offline-queue wait as well as execution. With commandTimeout below
connectTimeout, any handshake slower than the command deadline surfaced as
"Command timed out" from a Redis that never received the command.

- Raise the command deadline above the connect budget and derive both from
  named constants so the invariant cannot drift
- Give the PING health check its own deadline so the wider command budget
  does not slow failover detection
- Warm the shared connection at process start (Trigger.dev init, Next
  instrumentation) so a run's first command does not pay the handshake
  inside its own deadline
- Record an execution log when admission infrastructure is unreachable, so
  those runs show as failed instead of disappearing
- Re-admit rather than abort when a reservation refresh throws

Author: waleedlatif1

apps/sim/background/workflow-execution.ts

@@ -165,14 +165,32 @@ export async function executeWorkflowJob(
     const executionDeadlineAt = getExecutionDeadlineAt(timeoutController.signal)?.getTime()
     let admissionCompleted = payload.admissionCompleted === true
     if (admissionCompleted && executionDeadlineAt !== undefined) {
-      admissionCompleted = await refreshExecutionSlotExpiry(
-        executionId,
-        executionDeadlineAt + RESERVATION_TTL_BUFFER_MS
-      )
-      if (!admissionCompleted) {
-        logger.warn('Queued workflow reservation expired; repeating usage admission', {
+      /**
+       * A refresh that *returns* false already degrades into repeating usage
+       * admission below. A refresh that *throws* — a Redis timeout on the first
+       * command this fresh worker process issues — used to kill the run here,
+       * before the logging session exists, so the execution left no log row at
+       * all and simply vanished from the workspace's logs. Route both outcomes
+       * into the same re-admission path: preprocessing owns the reservation
+       * retry, and any failure it hits is recorded against a live session.
+       */
+      try {
+        admissionCompleted = await refreshExecutionSlotExpiry(
+          executionId,
+          executionDeadlineAt + RESERVATION_TTL_BUFFER_MS
+        )
+        if (!admissionCompleted) {
+          logger.warn('Queued workflow reservation expired; repeating usage admission', {
+            workflowId,
+            executionId,
+          })
+        }
+      } catch (error) {
+        admissionCompleted = false
+        logger.warn('Reservation refresh failed; repeating usage admission', {
           workflowId,
           executionId,
+          error: toError(error).message,
         })
       }
     }

apps/sim/instrumentation-node.ts

@@ -397,4 +397,14 @@ export async function register() {
 
   const { startMemoryTelemetry } = await import('./lib/monitoring/memory-telemetry')
   startMemoryTelemetry()
+
+  /**
+   * Open the shared Redis connection during boot so the first request does not
+   * pay the TLS handshake inside its own command deadline. Deliberately not
+   * awaited: a slow or unreachable Redis must never hold up serving, and
+   * `warmRedisConnection` already bounds and swallows its own failures.
+   */
+  void import('./lib/core/config/redis')
+    .then(({ warmRedisConnection }) => warmRedisConnection())
+    .catch((error) => logger.warn('Redis warm-up could not start', { error }))
 }

apps/sim/lib/core/config/redis.test.ts

@@ -29,8 +29,11 @@ import {
   closeRedisConnection,
   extendLock,
   getRedisClient,
+  getRedisConnectionDefaults,
   onRedisReconnect,
+  REDIS_COMMAND_TIMEOUT_MS,
   resetForTesting,
+  warmRedisConnection,
 } from '@/lib/core/config/redis'
 
 describe('redis config', () => {
@@ -303,6 +306,110 @@ describe('redis config', () => {
     })
   })
 
+  describe('command timeout', () => {
+    it('stays above the connect timeout so a slow handshake is not reported as a command timeout', () => {
+      const { connectTimeout } = getRedisConnectionDefaults('redis://localhost:6379')
+
+      expect(connectTimeout).toBeDefined()
+      expect(REDIS_COMMAND_TIMEOUT_MS).toBeGreaterThan(connectTimeout as number)
+    })
+
+    it('applies that timeout to the shared client', () => {
+      getRedisClient()
+
+      expect(MockRedisConstructor).toHaveBeenCalledWith(
+        expect.any(String),
+        expect.objectContaining({ commandTimeout: REDIS_COMMAND_TIMEOUT_MS })
+      )
+    })
+
+    it('does not let the command deadline govern how fast a dead connection is detected', async () => {
+      const listener = vi.fn()
+      onRedisReconnect(listener)
+      getRedisClient()
+
+      // A PING that never settles — the failure mode the health check exists for.
+      mockRedisInstance.ping.mockReturnValue(new Promise(() => {}))
+
+      // Two intervals plus two probe deadlines is well under two command
+      // deadlines, so this only passes while the probe has its own budget.
+      await vi.advanceTimersByTimeAsync(15_000)
+      await vi.advanceTimersByTimeAsync(15_000)
+      await vi.advanceTimersByTimeAsync(15_000)
+
+      expect(listener).toHaveBeenCalledTimes(1)
+      expect(3 * 15_000).toBeLessThan(2 * REDIS_COMMAND_TIMEOUT_MS + 2 * 15_000)
+    })
+  })
+
+  describe('warmRedisConnection', () => {
+    it('resolves without waiting when the client is already connected', async () => {
+      mockRedisInstance.status = 'ready'
+
+      await expect(warmRedisConnection()).resolves.toBeUndefined()
+      expect(mockRedisInstance.once).not.toHaveBeenCalled()
+    })
+
+    it('resolves once the connection reports ready', async () => {
+      mockRedisInstance.status = 'connecting'
+      const readyHandlers: Array<() => void> = []
+      mockRedisInstance.once.mockImplementation((event: string, cb: () => void) => {
+        if (event === 'ready') readyHandlers.push(cb)
+      })
+
+      let settled = false
+      const warm = warmRedisConnection().then(() => {
+        settled = true
+      })
+
+      await vi.advanceTimersByTimeAsync(2_000)
+      expect(settled).toBe(false)
+
+      for (const handler of readyHandlers) handler()
+      await warm
+
+      expect(settled).toBe(true)
+    })
+
+    it('gives up at the connect deadline rather than blocking startup forever', async () => {
+      mockRedisInstance.status = 'connecting'
+      mockRedisInstance.once.mockImplementation(() => {})
+
+      let settled = false
+      const warm = warmRedisConnection().then(() => {
+        settled = true
+      })
+
+      await vi.advanceTimersByTimeAsync(9_000)
+      expect(settled).toBe(false)
+
+      await vi.advanceTimersByTimeAsync(2_000)
+      await warm
+
+      expect(settled).toBe(true)
+    })
+
+    it('warms a given client once so a warm process pays nothing per unit of work', async () => {
+      mockRedisInstance.status = 'connecting'
+      mockRedisInstance.once.mockImplementation((event: string, cb: () => void) => {
+        if (event === 'ready') cb()
+      })
+
+      await warmRedisConnection()
+      const callsAfterFirst = mockRedisInstance.once.mock.calls.length
+
+      await warmRedisConnection()
+
+      expect(mockRedisInstance.once.mock.calls.length).toBe(callsAfterFirst)
+    })
+
+    it('resolves instead of throwing when Redis is not configured', async () => {
+      mockEnv.REDIS_URL = undefined
+
+      await expect(warmRedisConnection()).resolves.toBeUndefined()
+    })
+  })
+
   describe('retryStrategy', () => {
     function captureRetryStrategy(): (times: number) => number {
       let capturedConfig: Record<string, unknown> = {}

apps/sim/lib/core/config/redis.ts

@@ -36,18 +36,38 @@ function resolveRedisTlsOptions(url: string | undefined): { servername: string }
   return { servername: env.REDIS_TLS_SERVERNAME }
 }
 
+const REDIS_CONNECT_TIMEOUT_MS = 10_000
+
+/**
+ * Per-command deadline. MUST stay greater than `REDIS_CONNECT_TIMEOUT_MS`.
+ *
+ * `sendCommand` arms this timer *before* it checks whether the socket is
+ * writable and before the `enableOfflineQueue` branch, so a command issued
+ * while the connection is still being established is already counting down
+ * while it waits in the offline queue. Set below the connect timeout, every
+ * slow handshake surfaces as `Command timed out` — attributed to a Redis that
+ * never received the command, with a stack containing only ioredis timer
+ * frames.
+ *
+ * Production handshakes to ElastiCache measure 2-6s when several connections
+ * are opened at once, so a 5s command timeout failed roughly 3% of cold-start
+ * commands on Trigger.dev workers, which open a fresh connection per run.
+ */
+export const REDIS_COMMAND_TIMEOUT_MS = 15_000
+
 /**
  * Shared connection defaults — keepAlive, connectTimeout, enableOfflineQueue,
  * and TLS SNI when REDIS_URL targets an IP. Every Redis client we open should
- * spread this; callers add their own retry / timeout policy on top.
+ * spread this; callers add their own retry policy on top and take their command
+ * deadline from `REDIS_COMMAND_TIMEOUT_MS` so the invariant above holds.
  */
 export function getRedisConnectionDefaults(
   url: string | undefined
 ): Pick<RedisOptions, 'keepAlive' | 'connectTimeout' | 'enableOfflineQueue' | 'tls'> {
   const tls = resolveRedisTlsOptions(url)
   return {
     keepAlive: 1000,
-    connectTimeout: 10000,
+    connectTimeout: REDIS_CONNECT_TIMEOUT_MS,
     enableOfflineQueue: true,
     ...(tls ? { tls } : {}),
   }
@@ -59,6 +79,7 @@ interface RedisState {
   pingInterval: NodeJS.Timeout | null
   pingInFlight: boolean
   reconnectListeners: Array<() => void>
+  warmPromise: Promise<void> | null
 }
 
 const g = globalThis as typeof globalThis & { _redisState?: RedisState }
@@ -69,13 +90,46 @@ if (!g._redisState) {
     pingInterval: null,
     pingInFlight: false,
     reconnectListeners: [],
+    warmPromise: null,
   }
 }
 const state = g._redisState
 
 const PING_INTERVAL_MS = 15_000
 const MAX_PING_FAILURES = 2
 
+/**
+ * Deadline for a single health probe.
+ *
+ * A PING is only ever issued on an already-established connection, so unlike a
+ * general command it is never waiting on a handshake and takes a much tighter
+ * deadline. Keeping it independent of `REDIS_COMMAND_TIMEOUT_MS` is what stops
+ * the wider command deadline from slowing failover: two consecutive misses
+ * still force a reconnect within roughly two intervals.
+ */
+const REDIS_PING_TIMEOUT_MS = 5_000
+
+/**
+ * `commandTimeout` cannot express "probe deadline" separately from "command
+ * deadline" — it is a single client-wide option — so the probe carries its own.
+ */
+async function pingWithDeadline(redis: Redis): Promise<void> {
+  let timer: NodeJS.Timeout | undefined
+  try {
+    await Promise.race([
+      redis.ping(),
+      new Promise<never>((_, reject) => {
+        timer = setTimeout(
+          () => reject(new Error('Redis PING deadline exceeded')),
+          REDIS_PING_TIMEOUT_MS
+        )
+      }),
+    ])
+  } finally {
+    if (timer) clearTimeout(timer)
+  }
+}
+
 export function getConfiguredRedisUrl(): string | null {
   if (getConfiguredCacheProvider() === 'database') return null
 
@@ -101,7 +155,7 @@ function startPingHealthCheck(redis: Redis): void {
     if (state.pingInFlight) return
     state.pingInFlight = true
     try {
-      await redis.ping()
+      await pingWithDeadline(redis)
       state.pingFailures = 0
     } catch (error) {
       state.pingFailures++
@@ -117,6 +171,8 @@ function startPingHealthCheck(redis: Redis): void {
         state.pingFailures = 0
         // Clear before notifying listeners — they may call getRedisClient() and must see the reset state.
         state.client = null
+        // The next client is cold again, so let it be warmed before first use.
+        state.warmPromise = null
         if (state.pingInterval) {
           clearInterval(state.pingInterval)
           state.pingInterval = null
@@ -161,7 +217,7 @@ export function getRedisClient(): Redis | null {
 
     state.client = new Redis(redisUrl, {
       ...defaults,
-      commandTimeout: 5000,
+      commandTimeout: REDIS_COMMAND_TIMEOUT_MS,
       maxRetriesPerRequest: 5,
 
       retryStrategy: (times) => {
@@ -199,6 +255,78 @@ export function getRedisClient(): Redis | null {
   }
 }
 
+/**
+ * Establish the shared connection before the first command needs it.
+ *
+ * `commandTimeout` is a total deadline that starts the moment a command is
+ * issued: ioredis arms it in `sendCommand` before it checks whether the socket
+ * is writable, so the budget covers handshake and offline-queue wait as well as
+ * execution. A process whose first command lands on a cold client therefore
+ * spends most of that budget on the TLS handshake, which measures 2-6s against
+ * ElastiCache when several connections open at once.
+ *
+ * Warming at process start moves that cost off the first command's clock, which
+ * is the connection-reuse practice AWS recommends: establishing a TCP+TLS
+ * connection is far more expensive than the commands that run over it, so it
+ * should be paid once per process rather than once per unit of work.
+ *
+ * Best effort by design — resolves rather than rejects on failure, and is
+ * bounded by the connect budget, so neither a degraded Redis nor a missing
+ * configuration can stop a process from starting. Callers that skip warming
+ * still work; they just pay the handshake on their first command as before.
+ *
+ * Memoized per client: a warm process returns immediately, and a forced
+ * reconnect clears it so the replacement connection is warmed in turn.
+ */
+export function warmRedisConnection(): Promise<void> {
+  if (state.warmPromise) return state.warmPromise
+
+  let client: Redis | null
+  try {
+    client = getRedisClient()
+  } catch (error) {
+    logger.warn('Skipping Redis warm-up: client unavailable', {
+      error: toError(error).message,
+    })
+    return Promise.resolve()
+  }
+
+  if (!client) return Promise.resolve()
+  if (client.status === 'ready') return Promise.resolve()
+
+  const startedAt = Date.now()
+  state.warmPromise = new Promise<void>((resolve) => {
+    let settled = false
+    let timer: NodeJS.Timeout | undefined
+
+    const finish = (outcome: 'ready' | 'deadline') => {
+      if (settled) return
+      settled = true
+      if (timer) clearTimeout(timer)
+      client.off('ready', onReady)
+      const elapsedMs = Date.now() - startedAt
+      if (outcome === 'ready') {
+        logger.info('Redis connection warmed', { elapsedMs })
+      } else {
+        logger.warn('Redis warm-up did not complete before its deadline', { elapsedMs })
+      }
+      resolve()
+    }
+
+    const onReady = () => finish('ready')
+
+    /**
+     * Only `ready` settles early. A transient `error` is followed by ioredis's
+     * own retry, so resolving on it would hand back a still-cold client and
+     * reintroduce the very race this removes; the deadline is the backstop.
+     */
+    timer = setTimeout(() => finish('deadline'), REDIS_CONNECT_TIMEOUT_MS)
+    client.once('ready', onReady)
+  })
+
+  return state.warmPromise
+}
+
 /**
  * Lua script for safe lock release.
  * Only deletes the key if the value matches (ownership verification).
@@ -354,5 +482,6 @@ export function resetForTesting(): void {
   state.client = null
   state.pingFailures = 0
   state.pingInFlight = false
+  state.warmPromise = null
   state.reconnectListeners.length = 0
 }

apps/sim/lib/execution/execution-signal.test.ts

@@ -25,6 +25,7 @@ vi.mock('ioredis', () => ({
 vi.mock('@/lib/core/config/redis', () => ({
   getConfiguredRedisUrl: () => mockRedisUrl.value,
   getRedisConnectionDefaults: () => ({}),
+  REDIS_COMMAND_TIMEOUT_MS: 15_000,
 }))
 
 import {