feat(github): add retry-with-backoff logic for repository update failures

apps/backend/src/lib/retry/exponential-backoff.test.ts

@@ -0,0 +1,196 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import {
+    isRetryableError,
+    calculateBackoffDelay,
+    retryWithBackoff,
+    sleep,
+} from './exponential-backoff';
+
+describe('exponential-backoff', () => {
+    beforeEach(() => {
+        vi.useFakeTimers();
+    });
+
+    afterEach(() => {
+        vi.useRealTimers();
+    });
+
+    describe('isRetryableError', () => {
+        it('should return true for 5xx errors', () => {
+            expect(isRetryableError({ status: 500 })).toBe(true);
+            expect(isRetryableError({ status: 502 })).toBe(true);
+            expect(isRetryableError({ status: 503 })).toBe(true);
+        });
+
+        it('should return true for 429 rate limit', () => {
+            expect(isRetryableError({ status: 429 })).toBe(true);
+        });
+
+        it('should return false for 4xx errors (except 429)', () => {
+            expect(isRetryableError({ status: 400 })).toBe(false);
+            expect(isRetryableError({ status: 401 })).toBe(false);
+            expect(isRetryableError({ status: 403 })).toBe(false);
+            expect(isRetryableError({ status: 404 })).toBe(false);
+        });
+
+        it('should return true for network errors', () => {
+            expect(isRetryableError({ message: 'network error' })).toBe(true);
+            expect(isRetryableError({ message: 'ECONNREFUSED' })).toBe(true);
+            expect(isRetryableError({ message: 'timeout' })).toBe(true);
+            expect(isRetryableError({ message: 'ETIMEDOUT' })).toBe(true);
+        });
+
+        it('should return false for non-network errors', () => {
+            expect(isRetryableError({ message: 'validation error' })).toBe(false);
+            expect(isRetryableError({ message: 'invalid payload' })).toBe(false);
+        });
+
+        it('should return false for null/undefined', () => {
+            expect(isRetryableError(null)).toBe(false);
+            expect(isRetryableError(undefined)).toBe(false);
+        });
+    });
+
+    describe('calculateBackoffDelay', () => {
+        it('should increase exponentially', () => {
+            const delay0 = calculateBackoffDelay(0, 100, 10000, 2);
+            const delay1 = calculateBackoffDelay(1, 100, 10000, 2);
+            const delay2 = calculateBackoffDelay(2, 100, 10000, 2);
+
+            // With jitter, we can only check approximate values
+            expect(delay1).toBeGreaterThan(delay0);
+            expect(delay2).toBeGreaterThan(delay1);
+        });
+
+        it('should respect max delay', () => {
+            const delay = calculateBackoffDelay(10, 100, 500, 2);
+            expect(delay).toBeLessThanOrEqual(550); // 500 + 10% jitter
+        });
+
+        it('should add jitter', () => {
+            // Run multiple times to verify jitter is applied
+            const delays = Array.from({ length: 10 }, () =>
+                calculateBackoffDelay(1, 100, 1000, 2),
+            );
+
+            // Check that not all delays are identical (jitter applied)
+            const uniqueDelays = new Set(delays);
+            expect(uniqueDelays.size).toBeGreaterThan(1);
+        });
+
+        it('should return non-negative delay', () => {
+            const delay = calculateBackoffDelay(5, 100, 1000, 2);
+            expect(delay).toBeGreaterThanOrEqual(0);
+        });
+    });
+
+    describe('retryWithBackoff', () => {
+        it('should return success on first attempt', async () => {
+            const fn = vi.fn().mockResolvedValue('success');
+            const result = await retryWithBackoff(fn);
+
+            expect(result.success).toBe(true);
+            if (result.success) {
+                expect(result.data).toBe('success');
+                expect(result.attempts).toBe(1);
+            }
+            expect(fn).toHaveBeenCalledOnce();
+        });
+
+        it('should retry on retryable errors', async () => {
+            const fn = vi
+                .fn()
+                .mockRejectedValueOnce({ status: 503 })
+                .mockRejectedValueOnce({ status: 503 })
+                .mockResolvedValueOnce('success');
+
+            const result = await retryWithBackoff(fn, {
+                maxAttempts: 5,
+                initialDelayMs: 10,
+                maxDelayMs: 100,
+            });
+
+            expect(result.success).toBe(true);
+            if (result.success) {
+                expect(result.data).toBe('success');
+                expect(result.attempts).toBe(3);
+            }
+            expect(fn).toHaveBeenCalledTimes(3);
+        });
+
+        it('should not retry on non-retryable errors', async () => {
+            const fn = vi.fn().mockRejectedValue({ status: 400 });
+
+            const result = await retryWithBackoff(fn, { maxAttempts: 5 });
+
+            expect(result.success).toBe(false);
+            if (!result.success) {
+                expect(result.attempts).toBe(1);
+            }
+            expect(fn).toHaveBeenCalledOnce();
+        });
+
+        it('should respect max attempts', async () => {
+            const fn = vi.fn().mockRejectedValue({ status: 503 });
+
+            const result = await retryWithBackoff(fn, { maxAttempts: 3, initialDelayMs: 10 });
+
+            expect(result.success).toBe(false);
+            if (!result.success) {
+                expect(result.attempts).toBe(3);
+            }
+            expect(fn).toHaveBeenCalledTimes(3);
+        });
+
+        it('should respect max total duration', async () => {
+            const fn = vi.fn().mockRejectedValue({ status: 503 });
+
+            const result = await retryWithBackoff(fn, {
+                maxAttempts: 10,
+                initialDelayMs: 100,
+                maxDelayMs: 100,
+                maxTotalDurationMs: 250,
+            });
+
+            expect(result.success).toBe(false);
+            if (!result.success) {
+                expect(result.totalDurationMs).toBeLessThan(500);
+            }
+        });
+
+        it('should handle async errors', async () => {
+            const fn = vi
+                .fn()
+                .mockRejectedValueOnce(new Error('network timeout'))
+                .mockResolvedValueOnce('success');
+
+            const result = await retryWithBackoff(fn, { maxAttempts: 3, initialDelayMs: 10 });
+
+            expect(result.success).toBe(true);
+            if (result.success) {
+                expect(result.attempts).toBe(2);
+            }
+        });
+
+        it('should track total duration', async () => {
+            const fn = vi.fn().mockResolvedValue('success');
+
+            const result = await retryWithBackoff(fn, { initialDelayMs: 10 });
+
+            expect(result.success).toBe(true);
+            if (result.success) {
+                expect(result.totalDurationMs).toBeDefined();
+            }
+        });
+    });
+
+    describe('sleep', () => {
+        it('should resolve after delay', async () => {
+            const start = Date.now();
+            await sleep(100);
+            vi.runAllTimers();
+            const elapsed = Date.now() - start;
+            expect(elapsed).toBeGreaterThanOrEqual(100);
+        });
+    });
+});

apps/backend/src/lib/retry/exponential-backoff.ts

@@ -0,0 +1,179 @@
+/**
+ * Exponential backoff retry with jitter for transient failures.
+ * Distinguishes retryable (5xx, network, rate limit) from non-retryable (4xx) errors.
+ */
+
+export interface RetryOptions {
+    maxAttempts?: number;
+    initialDelayMs?: number;
+    maxDelayMs?: number;
+    maxTotalDurationMs?: number;
+    backoffMultiplier?: number;
+}
+
+export interface RetryResult<T> {
+    success: true;
+    data: T;
+    attempts: number;
+} | {
+    success: false;
+    error: Error;
+    attempts: number;
+    totalDurationMs: number;
+}
+
+const DEFAULT_MAX_ATTEMPTS = 5;
+const DEFAULT_INITIAL_DELAY_MS = 100;
+const DEFAULT_MAX_DELAY_MS = 30000; // 30 seconds
+const DEFAULT_MAX_TOTAL_DURATION_MS = 5 * 60 * 1000; // 5 minutes
+const DEFAULT_BACKOFF_MULTIPLIER = 2;
+
+/**
+ * Check if an error is retryable
+ * Retryable errors: 5xx, 429 (rate limit), network errors, timeout
+ * Non-retryable: 4xx (except 429), auth errors, validation errors
+ */
+export function isRetryableError(error: unknown): boolean {
+    if (!error) return false;
+
+    // Check for HTTP status codes
+    if (typeof (error as any).status === 'number') {
+        const status = (error as any).status;
+        // Retry on 5xx and 429 (rate limit)
+        if (status >= 500 || status === 429) return true;
+        // Don't retry on 4xx (except 429)
+        if (status >= 400 && status < 500) return false;
+    }
+
+    // Check for network/timeout errors
+    const message = (error as any).message as string;
+    if (!message) return false;
+
+    const networkPatterns = [
+        /network/i,
+        /timeout/i,
+        /econnrefused/i,
+        /econnreset/i,
+        /enotfound/i,
+        /eai_again/i,
+        /socket hang up/i,
+        /ETIMEDOUT/i,
+        /EHOSTUNREACH/i,
+    ];
+
+    return networkPatterns.some((p) => p.test(message));
+}
+
+/**
+ * Calculate backoff delay with jitter
+ * Prevents thundering herd by adding random jitter (±10%)
+ */
+export function calculateBackoffDelay(
+    attempt: number,
+    initialDelayMs: number,
+    maxDelayMs: number,
+    multiplier: number,
+): number {
+    // Exponential backoff: initial * (multiplier ^ attempt)
+    let delay = initialDelayMs * Math.pow(multiplier, attempt);
+
+    // Cap at max delay
+    delay = Math.min(delay, maxDelayMs);
+
+    // Add jitter: ±10% of the delay
+    const jitter = delay * 0.1 * (Math.random() - 0.5) * 2;
+    return Math.max(0, delay + jitter);
+}
+
+/**
+ * Sleep for a given number of milliseconds
+ */
+export function sleep(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Retry a function with exponential backoff
+ *
+ * @param fn - Async function to retry
+ * @param options - Retry configuration
+ * @returns Result with data on success or error on failure
+ */
+export async function retryWithBackoff<T>(
+    fn: () => Promise<T>,
+    options: RetryOptions = {},
+): Promise<RetryResult<T>> {
+    const {
+        maxAttempts = DEFAULT_MAX_ATTEMPTS,
+        initialDelayMs = DEFAULT_INITIAL_DELAY_MS,
+        maxDelayMs = DEFAULT_MAX_DELAY_MS,
+        maxTotalDurationMs = DEFAULT_MAX_TOTAL_DURATION_MS,
+        backoffMultiplier = DEFAULT_BACKOFF_MULTIPLIER,
+    } = options;
+
+    let lastError: Error | undefined;
+    const startTime = Date.now();
+
+    for (let attempt = 0; attempt < maxAttempts; attempt++) {
+        try {
+            const data = await fn();
+            return { success: true, data, attempts: attempt + 1 };
+        } catch (err: unknown) {
+            lastError = err instanceof Error ? err : new Error(String(err));
+
+            // Check if error is retryable
+            if (!isRetryableError(err)) {
+                return {
+                    success: false,
+                    error: lastError,
+                    attempts: attempt + 1,
+                    totalDurationMs: Date.now() - startTime,
+                };
+            }
+
+            // Check if we've exceeded total duration
+            const elapsedMs = Date.now() - startTime;
+            if (elapsedMs >= maxTotalDurationMs) {
+                return {
+                    success: false,
+                    error: new Error(
+                        `Retry exhausted: max total duration of ${maxTotalDurationMs}ms exceeded`,
+                    ),
+                    attempts: attempt + 1,
+                    totalDurationMs: elapsedMs,
+                };
+            }
+
+            // Calculate delay for next attempt
+            if (attempt < maxAttempts - 1) {
+                const delayMs = calculateBackoffDelay(attempt, initialDelayMs, maxDelayMs, backoffMultiplier);
+                await sleep(delayMs);
+            }
+        }
+    }
+
+    return {
+        success: false,
+        error: lastError || new Error('Unknown error'),
+        attempts: maxAttempts,
+        totalDurationMs: Date.now() - startTime,
+    };
+}
+
+/**
+ * Helper to create a retryable function that tracks retry state
+ */
+export class RetryableOperation<T> {
+    constructor(
+        private fn: () => Promise<T>,
+        private options: RetryOptions = {},
+    ) {}
+
+    async execute(): Promise<T> {
+        const result = await retryWithBackoff(this.fn, this.options);
+        if (!result.success) {
+            throw result.error;
+        }
+        return result.data;
+    }
+}