Skip to content

Commit fca7848

Browse files
icecrasher321claude
andcommitted
improvement(webhooks): stop redoing ingest work in the same-process worker
preprocessExecution gains trustWorkflowRecord (skip the archived-state re-read for a row fetched in the same request) and skipAccountChecks (skip the ban + subscription re-reads; guarded to checkRateLimit: false). Webhook ingest passes trust for the row findAllWebhooksForPath returned; the inline runner closures hand the ingest-loaded workflow + webhook rows to executeWebhookJob as memory-only warm context, so the worker drops the third workflow fetch, the webhook re-select, and the duplicate account checks. Trigger.dev and recovery jobs pass no warm context and are byte-identical to before. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent bb0bd9f commit fca7848

6 files changed

Lines changed: 344 additions & 12 deletions

File tree

apps/sim/background/webhook-execution.test.ts

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -351,6 +351,154 @@ describe('executeWebhookJob fault vs error handling', () => {
351351
)
352352
})
353353

354+
it('reuses ingest-loaded rows and skips duplicate account checks with warm context', async () => {
355+
mockExecuteWorkflowCore.mockResolvedValue({
356+
success: true,
357+
status: 'completed',
358+
output: {},
359+
logs: [],
360+
executionState: {
361+
blockStates: {},
362+
executedBlocks: [],
363+
blockLogs: [],
364+
decisions: {},
365+
completedLoops: [],
366+
activeExecutionPath: [],
367+
},
368+
})
369+
const warmContext = {
370+
workflowRecord: { id: 'workflow-1', workspaceId: 'workspace-1', userId: 'user-1' },
371+
webhookRecord: { id: 'webhook-1', providerConfig: { warm: true } },
372+
} as unknown as NonNullable<Parameters<typeof executeWebhookJob>[2]>
373+
374+
await executeWebhookJob(payload, undefined, warmContext)
375+
376+
expect(executionPreprocessingMockFns.mockPreprocessExecution).toHaveBeenCalledWith(
377+
expect.objectContaining({
378+
workflowRecord: expect.objectContaining({ id: 'workflow-1' }),
379+
trustWorkflowRecord: true,
380+
skipAccountChecks: true,
381+
})
382+
)
383+
expect(dbChainMockFns.limit).not.toHaveBeenCalled()
384+
expect(mockResolveWebhookRecordProviderConfig).toHaveBeenCalledWith(
385+
expect.objectContaining({ id: 'webhook-1', providerConfig: { warm: true } }),
386+
'user-1',
387+
'workspace-1',
388+
expect.any(Object)
389+
)
390+
})
391+
392+
it('loads rows and keeps account checks without warm context', async () => {
393+
mockExecuteWorkflowCore.mockResolvedValue({
394+
success: true,
395+
status: 'completed',
396+
output: {},
397+
logs: [],
398+
executionState: {
399+
blockStates: {},
400+
executedBlocks: [],
401+
blockLogs: [],
402+
decisions: {},
403+
completedLoops: [],
404+
activeExecutionPath: [],
405+
},
406+
})
407+
408+
await executeWebhookJob(payload)
409+
410+
expect(executionPreprocessingMockFns.mockPreprocessExecution).toHaveBeenCalledWith(
411+
expect.objectContaining({
412+
workflowRecord: undefined,
413+
trustWorkflowRecord: false,
414+
skipAccountChecks: false,
415+
})
416+
)
417+
expect(dbChainMockFns.limit).toHaveBeenCalled()
418+
})
419+
420+
it('ignores warm rows whose ids do not match the payload', async () => {
421+
mockExecuteWorkflowCore.mockResolvedValue({
422+
success: true,
423+
status: 'completed',
424+
output: {},
425+
logs: [],
426+
executionState: {
427+
blockStates: {},
428+
executedBlocks: [],
429+
blockLogs: [],
430+
decisions: {},
431+
completedLoops: [],
432+
activeExecutionPath: [],
433+
},
434+
})
435+
const warmContext = {
436+
workflowRecord: { id: 'other-workflow' },
437+
webhookRecord: { id: 'other-webhook' },
438+
} as unknown as NonNullable<Parameters<typeof executeWebhookJob>[2]>
439+
440+
await executeWebhookJob(payload, undefined, warmContext)
441+
442+
expect(executionPreprocessingMockFns.mockPreprocessExecution).toHaveBeenCalledWith(
443+
expect.objectContaining({
444+
workflowRecord: undefined,
445+
trustWorkflowRecord: false,
446+
skipAccountChecks: false,
447+
})
448+
)
449+
expect(dbChainMockFns.limit).toHaveBeenCalled()
450+
})
451+
452+
it('logs phase timings and the executor-start metric for latency-tracked payloads', async () => {
453+
mockExecuteWorkflowCore.mockResolvedValue({
454+
success: true,
455+
status: 'completed',
456+
output: {},
457+
logs: [],
458+
executionState: {
459+
blockStates: {},
460+
executedBlocks: [],
461+
blockLogs: [],
462+
decisions: {},
463+
completedLoops: [],
464+
activeExecutionPath: [],
465+
},
466+
})
467+
468+
await executeWebhookJob({
469+
...payload,
470+
webhookReceivedAt: Date.now() - 100,
471+
triggerTimestampMs: Date.now() - 500,
472+
})
473+
474+
expect(webhookExecutionLogger.info).toHaveBeenCalledWith(
475+
'[request-1] Webhook dispatch latency',
476+
expect.objectContaining({
477+
dispatchLatencyMs: expect.any(Number),
478+
triggerAgeMs: expect.any(Number),
479+
preprocessMs: expect.any(Number),
480+
loadsMs: expect.any(Number),
481+
providerConfigMs: expect.any(Number),
482+
formatInputMs: expect.any(Number),
483+
})
484+
)
485+
486+
const coreOptions = mockExecuteWorkflowCore.mock.calls[0]?.[0] as {
487+
callbacks: { onBlockStart?: () => Promise<void> }
488+
}
489+
expect(coreOptions.callbacks.onBlockStart).toBeTypeOf('function')
490+
await coreOptions.callbacks.onBlockStart?.()
491+
await coreOptions.callbacks.onBlockStart?.()
492+
const executorStartCalls = webhookExecutionLogger.info.mock.calls.filter(
493+
([message]: [string]) => String(message).includes('Webhook executor started')
494+
)
495+
expect(executorStartCalls).toHaveLength(1)
496+
expect(executorStartCalls[0]?.[1]).toMatchObject({
497+
executorStartLatencyMs: expect.any(Number),
498+
executorStartTriggerAgeMs: expect.any(Number),
499+
})
500+
})
501+
354502
it('does not pass provider-config provenance absent from the trigger input', async () => {
355503
mockGetEffectiveEnvironmentSnapshot.mockResolvedValue({
356504
personalEncrypted: { WEBHOOK_SECRET: 'personal-ciphertext' },

apps/sim/background/webhook-execution.ts

Lines changed: 28 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { db } from '@sim/db'
2-
import { account, webhook } from '@sim/db/schema'
2+
import { account, webhook, type workflow } from '@sim/db/schema'
33
import { createLogger, runWithRequestContext } from '@sim/logger'
44
import { toError } from '@sim/utils/errors'
55
import { generateId } from '@sim/utils/id'
@@ -291,9 +291,21 @@ export type WebhookExecutionPayload = {
291291
executionTimeoutMs?: number
292292
}
293293

294+
/**
295+
* Memory-only rows the same-process inline runner hands over so the worker does
296+
* not re-read what ingest just loaded. Never serialized into the persisted job
297+
* payload — Trigger.dev and recovery paths run without it and load everything
298+
* themselves.
299+
*/
300+
export interface WebhookWarmContext {
301+
workflowRecord?: typeof workflow.$inferSelect
302+
webhookRecord?: typeof webhook.$inferSelect
303+
}
304+
294305
export async function executeWebhookJob(
295306
payload: WebhookExecutionPayload,
296-
externalAbortSignal?: AbortSignal
307+
externalAbortSignal?: AbortSignal,
308+
warmContext?: WebhookWarmContext
297309
) {
298310
const correlation = buildWebhookCorrelation(payload)
299311
const executionId = correlation.executionId
@@ -358,7 +370,8 @@ export async function executeWebhookJob(
358370
payload,
359371
correlation,
360372
timeoutController,
361-
admissionCompleted
373+
admissionCompleted,
374+
warmContext
362375
)
363376
}
364377

@@ -482,7 +495,8 @@ async function executeWebhookJobInternal(
482495
payload: WebhookExecutionPayload,
483496
correlation: AsyncExecutionCorrelation,
484497
timeoutController: ReturnType<typeof createTimeoutAbortController>,
485-
admissionCompleted: boolean
498+
admissionCompleted: boolean,
499+
warmContext?: WebhookWarmContext
486500
) {
487501
const { executionId, requestId } = correlation
488502
const loggingSession = new LoggingSession(
@@ -493,6 +507,8 @@ async function executeWebhookJobInternal(
493507
)
494508
loggingSession.setExecutionDeadlineAt(getExecutionDeadlineAt(timeoutController.signal))
495509

510+
const warmWorkflowRecord =
511+
warmContext?.workflowRecord?.id === payload.workflowId ? warmContext.workflowRecord : undefined
496512
const preprocessStartedAt = Date.now()
497513
const preprocessResult = await preprocessExecution({
498514
workflowId: payload.workflowId,
@@ -509,6 +525,9 @@ async function executeWebhookJobInternal(
509525
billingAttribution: payload.billingAttribution,
510526
executionType: 'async',
511527
executionDeadlineAt: getExecutionDeadlineAt(timeoutController.signal)?.getTime(),
528+
workflowRecord: warmWorkflowRecord,
529+
trustWorkflowRecord: Boolean(warmWorkflowRecord),
530+
skipAccountChecks: admissionCompleted && Boolean(warmWorkflowRecord),
512531
})
513532
const preprocessEndedAt = Date.now()
514533

@@ -559,9 +578,13 @@ async function executeWebhookJobInternal(
559578
workspaceId
560579
)
561580
: loadDeployedWorkflowState(payload.workflowId, workspaceId)
581+
const warmWebhookRecord =
582+
warmContext?.webhookRecord?.id === payload.webhookId ? warmContext.webhookRecord : undefined
562583
const [workflowData, webhookRows, resolvedCredentialUserId] = await Promise.all([
563584
workflowStatePromise,
564-
db.select().from(webhook).where(eq(webhook.id, payload.webhookId)).limit(1),
585+
warmWebhookRecord
586+
? Promise.resolve([warmWebhookRecord])
587+
: db.select().from(webhook).where(eq(webhook.id, payload.webhookId)).limit(1),
565588
payload.credentialId
566589
? resolveCredentialAccountUserId(payload.credentialId)
567590
: Promise.resolve(undefined),

apps/sim/lib/execution/preprocessing.test.ts

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,124 @@ describe('preprocessExecution deployment checks', () => {
145145
})
146146
})
147147

148+
describe('preprocessExecution prefetched record trust and account-check skips', () => {
149+
type PrefetchedRecord = NonNullable<Parameters<typeof preprocessExecution>[0]['workflowRecord']>
150+
const prefetchedRecord = {
151+
id: 'workflow-1',
152+
userId: 'creator-1',
153+
workspaceId: 'workspace-1',
154+
isDeployed: true,
155+
archivedAt: null,
156+
} as unknown as PrefetchedRecord
157+
158+
beforeEach(() => {
159+
vi.clearAllMocks()
160+
})
161+
162+
it('skips the active-record re-read when the prefetched record is trusted', async () => {
163+
const result = await preprocessExecution({
164+
workflowId: 'workflow-1',
165+
userId: 'user-1',
166+
triggerType: 'webhook',
167+
executionId: 'execution-1',
168+
requestId: 'request-1',
169+
checkRateLimit: false,
170+
workflowRecord: prefetchedRecord,
171+
trustWorkflowRecord: true,
172+
})
173+
174+
expect(result.success).toBe(true)
175+
expect(workflowAuthzMockFns.mockGetActiveWorkflowRecord).not.toHaveBeenCalled()
176+
})
177+
178+
it('re-reads the active record for a prefetched record without trust', async () => {
179+
const result = await preprocessExecution({
180+
workflowId: 'workflow-1',
181+
userId: 'user-1',
182+
triggerType: 'webhook',
183+
executionId: 'execution-1',
184+
requestId: 'request-1',
185+
checkRateLimit: false,
186+
workflowRecord: prefetchedRecord,
187+
})
188+
189+
expect(result.success).toBe(true)
190+
expect(workflowAuthzMockFns.mockGetActiveWorkflowRecord).toHaveBeenCalledWith('workflow-1')
191+
})
192+
193+
it('still rejects a trusted prefetched record that is archived', async () => {
194+
const result = await preprocessExecution({
195+
workflowId: 'workflow-1',
196+
userId: 'user-1',
197+
triggerType: 'webhook',
198+
executionId: 'execution-1',
199+
requestId: 'request-1',
200+
checkRateLimit: false,
201+
workflowRecord: {
202+
...prefetchedRecord,
203+
archivedAt: new Date('2026-08-01T00:00:00.000Z'),
204+
} as unknown as PrefetchedRecord,
205+
trustWorkflowRecord: true,
206+
})
207+
208+
expect(result).toEqual({
209+
success: false,
210+
error: { message: 'Workflow not found', statusCode: 404 },
211+
})
212+
expect(workflowAuthzMockFns.mockGetActiveWorkflowRecord).not.toHaveBeenCalled()
213+
})
214+
215+
it('skips the ban and subscription reads when skipAccountChecks is set', async () => {
216+
const result = await preprocessExecution({
217+
workflowId: 'workflow-1',
218+
userId: 'user-1',
219+
triggerType: 'webhook',
220+
executionId: 'execution-1',
221+
requestId: 'request-1',
222+
checkRateLimit: false,
223+
skipAccountChecks: true,
224+
workflowRecord: prefetchedRecord,
225+
trustWorkflowRecord: true,
226+
})
227+
228+
expect(result.success).toBe(true)
229+
if (result.success) {
230+
expect(result.actorSubscription).toBeNull()
231+
}
232+
expect(mockGetActivelyBannedUserIds).not.toHaveBeenCalled()
233+
expect(getHighestPrioritySubscription).not.toHaveBeenCalled()
234+
})
235+
236+
it('keeps the ban and subscription reads by default', async () => {
237+
const result = await preprocessExecution({
238+
workflowId: 'workflow-1',
239+
userId: 'user-1',
240+
triggerType: 'webhook',
241+
executionId: 'execution-1',
242+
requestId: 'request-1',
243+
checkRateLimit: false,
244+
})
245+
246+
expect(result.success).toBe(true)
247+
expect(mockGetActivelyBannedUserIds).toHaveBeenCalled()
248+
expect(getHighestPrioritySubscription).toHaveBeenCalled()
249+
})
250+
251+
it('rejects skipAccountChecks combined with rate limiting', async () => {
252+
await expect(
253+
preprocessExecution({
254+
workflowId: 'workflow-1',
255+
userId: 'user-1',
256+
triggerType: 'webhook',
257+
executionId: 'execution-1',
258+
requestId: 'request-1',
259+
checkRateLimit: true,
260+
skipAccountChecks: true,
261+
})
262+
).rejects.toThrow('skipAccountChecks requires checkRateLimit: false')
263+
})
264+
})
265+
148266
describe('preprocessExecution correlation logging', () => {
149267
it('preserves trigger correlation when logging preprocessing failures', async () => {
150268
mockResolveSystemBillingAttribution.mockRejectedValueOnce(

0 commit comments

Comments
 (0)