OUT-3487: add delayed retry backoff for trigger.dev sync tasks

src/trigger/processFileSync.ts

@@ -40,6 +40,25 @@ type HandleChannelFilePayload = {
 
 const machine = env.TRIGGER_MACHINE
 
+/**
+ * Retry config for per-file sync tasks (e.g., syncDropboxFileToAssembly, deleteDropboxFileInAssembly).
+ * Delays retries to give the DB time to recover from transient failures (connection issues, timeouts).
+ *
+ * - maxAttempts: Number of retry attempts after the initial failure.
+ * - minTimeoutInMs: Initial wait before the first retry (2 min).
+ * - maxTimeoutInMs: Upper cap on retry delay (5 min).
+ * - factor: Exponential backoff multiplier. Each retry waits factor × previous delay
+ *   (e.g., 2 min → 4 min → 5 min capped).
+ * - randomize: Adds jitter (±50%) to prevent multiple failed tasks from retrying simultaneously.
+ */
+const RETRY_CONFIG = {
+  maxAttempts: 3,
+  minTimeoutInMs: 120_000, // 2 minutes
+  maxTimeoutInMs: 300_000, // 5 minutes
+  factor: 2,
+  randomize: true,
+} as const
+
 export const processDropboxChanges = task({
   id: 'process-dropbox-changes',
   machine,
@@ -59,20 +78,24 @@ export const processDropboxChanges = task({
 export const bidirectionalMasterSync = task({
   id: 'bidirectional-master-sync',
   machine,
+  retry: {
+    maxAttempts: 0,
+  },
   run: async (payload: SyncTaskPayload) => {
-    try {
-      await initiateAssemblyToDropboxSync.triggerAndWait(payload)
-      logger.info('\n\n Synced Assembly files to Dropbox \n\n')
-      await initiateDropboxToAssemblySync.triggerAndWait(payload)
-    } catch (error: unknown) {
-      logger.error('processFileSync#bidirectionalMasterSync', { error })
-    }
+    await initiateAssemblyToDropboxSync.triggerAndWait(payload)
+    logger.info('Synced Assembly files to Dropbox')
+
+    await initiateDropboxToAssemblySync.triggerAndWait(payload)
+    logger.info('Synced Dropbox files to Assembly')
   },
 })
 
 export const initiateDropboxToAssemblySync = task({
   id: 'initiate-dropbox-to-assembly-sync',
   machine,
+  retry: {
+    maxAttempts: 0,
+  },
   run: async (payload: SyncTaskPayload) => {
     logger.info(
       'processFileSync#initiateDropboxToAssemblySync. Syncing files from Dropbox to Assembly',
@@ -154,9 +177,7 @@ export const syncDropboxFileToAssembly = task({
     name: 'sync-dropbox-file-to-assembly',
     concurrencyLimit: 25,
   },
-  retry: {
-    maxAttempts: 3,
-  },
+  retry: RETRY_CONFIG,
   run: (payload: DropboxToAssemblySyncFilesPayload) => {
     logger.info('processFileSync#syncDropboxFileToAssembly')
     return withErrorLogging<DropboxToAssemblySyncFilesPayload>(payload, async () => {
@@ -283,9 +304,7 @@ export const deleteDropboxFileInAssembly = task({
     name: 'delete-dropbox-file-in-assembly',
     concurrencyLimit: 1,
   },
-  retry: {
-    maxAttempts: 3,
-  },
+  retry: RETRY_CONFIG,
   run: async (payload: DropboxToAssemblySyncFilesPayload) => {
     const { opts, entry } = payload
     const { channelSyncId, user, connectionToken, dbxRootPath } = opts
@@ -300,9 +319,7 @@ export const updateDropboxFileInAssembly = task({
     name: 'update-dropbox-file-in-assembly',
     concurrencyLimit: 5,
   },
-  retry: {
-    maxAttempts: 3,
-  },
+  retry: RETRY_CONFIG,
   run: async (payload: DropboxToAssemblySyncFilesPayload) => {
     await deleteDropboxFileInAssembly.triggerAndWait(payload)
     await syncDropboxFileToAssembly.trigger(payload)
@@ -312,6 +329,9 @@ export const updateDropboxFileInAssembly = task({
 export const initiateAssemblyToDropboxSync = task({
   id: 'initiate-assembly-to-dropbox-sync',
   machine,
+  retry: {
+    maxAttempts: 0,
+  },
   run: async (payload: SyncTaskPayload) => {
     logger.info(
       'processFileSync#initiateAssemblyToDropboxSync. Syncing files from Assembly to Dropbox',
@@ -356,9 +376,7 @@ export const syncAssemblyFileToDropbox = task({
     name: 'sync-assembly-file-to-dropbox',
     concurrencyLimit: 25,
   },
-  retry: {
-    maxAttempts: 3,
-  },
+  retry: RETRY_CONFIG,
   run: (payload: AssemblyToDropboxSyncFilesPayload) => {
     logger.info('processFileSync#syncAssemblyFileToDropbox')
     return withErrorLogging<AssemblyToDropboxSyncFilesPayload>(payload, async () => {

trigger.config.ts

@@ -69,6 +69,10 @@ export default defineConfig({
   onFailure: ({ payload, error, ctx }) => {
     console.error({ payload, error, ctx })
     Sentry.captureException(error, {
+      tags: {
+        taskId: ctx.task.id,
+        attemptNumber: ctx.attempt.number,
+      },
       extra: {
         payload,
         ctx,