From d41383e5cd3d4b329f21deaf550c82d6b6b6a14e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 10 Nov 2026 10:18:23 +0000 Subject: [PATCH] feat(jobs): resume jobs after a restart A job running when its worker was killed stayed 'running' forever, and a graceful shutdown just stopped the timer, leaving the running job to the same fate. - The worker renews a running job's lock while its handler runs, and before claiming releases running jobs whose lock is older than JOB_LOCK_TTL_MS (10 min) back to pending. - On shutdown it claims nothing more, aborts the handler's new JobContext.signal and waits up to JOB_SHUTDOWN_WAIT_MS (25s). A handler that stops on the signal (throws) goes back to pending without using an attempt; claimed jobs not yet started are handed back. Handlers that ignore the new optional argument behave as before. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012GTictazJTxnndn2m4LbtH --- .../jobs/__tests__/job-worker.service.spec.ts | 80 ++++++++++++++++++- utils/jobs/__tests__/job.service.spec.ts | 25 ++++++ utils/jobs/job-worker.service.ts | 59 ++++++++++++-- utils/jobs/job.service.ts | 36 ++++++++- utils/jobs/job.types.ts | 14 +++- 5 files changed, 205 insertions(+), 9 deletions(-) diff --git a/utils/jobs/__tests__/job-worker.service.spec.ts b/utils/jobs/__tests__/job-worker.service.spec.ts index ef82041..1e2aab7 100644 --- a/utils/jobs/__tests__/job-worker.service.spec.ts +++ b/utils/jobs/__tests__/job-worker.service.spec.ts @@ -17,6 +17,9 @@ describe('JobWorkerService', () => { markFailed: jest.fn(), nextAvailableAt: jest.fn().mockResolvedValue(null), onEnqueue: jest.fn(), + releaseStale: jest.fn().mockResolvedValue(0), + release: jest.fn(), + heartbeat: jest.fn().mockResolvedValue(undefined), } as any; } @@ -33,7 +36,7 @@ describe('JobWorkerService', () => { worker = new JobWorkerService([handler], jobService, makeScheduleSync()); const job = { id: 1, type: 'demo', payload: {} } as Job; await worker.runJob(job); - expect(handler.handle).toHaveBeenCalledWith(job); + expect(handler.handle).toHaveBeenCalledWith(job, { signal: expect.any(AbortSignal) }); expect(jobService.markCompleted).toHaveBeenCalledWith(job, { summary: 'ok' }); }); @@ -115,6 +118,81 @@ describe('JobWorkerService', () => { expect(jobService.markCompleted).toHaveBeenCalledWith(job, null); }); + describe('restarts', () => { + it('releases stale running jobs of a dead worker before claiming', async () => { + process.env.WORKER_ENABLED = 'true'; + const jobService = makeJobService(); + worker = new JobWorkerService([], jobService, makeScheduleSync()); + await worker.poll(); + expect(jobService.releaseStale).toHaveBeenCalledWith(10 * 60_000); + expect(jobService.releaseStale.mock.invocationCallOrder[0]).toBeLessThan(jobService.claimNext.mock.invocationCallOrder[0]); + }); + + it('on shutdown aborts the handler, waits for it and puts its job back to pending', async () => { + process.env.WORKER_ENABLED = 'true'; + let started: () => void; + const running = new Promise((resolve) => (started = resolve)); + const handler: JobHandler = { + type: 'demo', + handle: jest.fn((_job, { signal }) => { + started(); + return new Promise((_resolve, reject) => signal.addEventListener('abort', () => setTimeout(() => reject(new Error('stopped')), 10))); + }), + }; + const job = { id: 10, type: 'demo', payload: {}, attempts: 1 } as Job; + const jobService = makeJobService(); + jobService.claimNext.mockResolvedValue([job]); + worker = new JobWorkerService([handler], jobService, makeScheduleSync()); + void worker.poll(); + await running; + await worker.onModuleDestroy(); + expect(jobService.release).toHaveBeenCalledWith(job); + expect(jobService.markFailed).not.toHaveBeenCalled(); + // Stopped: no more polls. + await worker.poll(); + expect(jobService.claimNext).toHaveBeenCalledTimes(1); + }); + + it('a job that finishes during the shutdown wait is completed', async () => { + process.env.WORKER_ENABLED = 'true'; + const handler: JobHandler = { type: 'demo', handle: jest.fn(() => new Promise((resolve) => setTimeout(() => resolve({ summary: 'ok' }), 20))) }; + const job = { id: 11, type: 'demo', payload: {} } as Job; + const jobService = makeJobService(); + jobService.claimNext.mockResolvedValue([job]); + worker = new JobWorkerService([handler], jobService, makeScheduleSync()); + void worker.poll(); + await new Promise((r) => setTimeout(r, 5)); + await worker.onModuleDestroy(); + expect(jobService.markCompleted).toHaveBeenCalledWith(job, { summary: 'ok' }); + expect(jobService.release).not.toHaveBeenCalled(); + }); + + it('hands back jobs claimed but not started when stopping', async () => { + const jobs = [{ id: 12, type: 'demo' }, { id: 13, type: 'demo' }] as Job[]; + const jobService = makeJobService(); + jobService.claimNext.mockResolvedValue(jobs); + let worker2: JobWorkerService; + const handler: JobHandler = { type: 'demo', handle: jest.fn(async () => { void worker2.onModuleDestroy(); }) }; + worker2 = worker = new JobWorkerService([handler], jobService, makeScheduleSync()); + await worker.processBatch(); + expect(handler.handle).toHaveBeenCalledTimes(1); + expect(jobService.markCompleted).toHaveBeenCalledWith(jobs[0], null); + expect(jobService.release).toHaveBeenCalledWith(jobs[1]); + }); + + it('renews the lock while the handler runs', async () => { + jest.useFakeTimers(); + const handler: JobHandler = { type: 'demo', handle: jest.fn(() => new Promise((resolve) => setTimeout(resolve, 6 * 60_000))) }; + const jobService = makeJobService(); + worker = new JobWorkerService([handler], jobService, makeScheduleSync()); + const run = worker.runJob({ id: 14, type: 'demo', payload: {} } as Job); + await jest.advanceTimersByTimeAsync(6 * 60_000); + await run; + expect(jobService.heartbeat).toHaveBeenCalledTimes(2); + jest.useRealTimers(); + }); + }); + describe('timer', () => { beforeEach(() => { jest.useFakeTimers(); diff --git a/utils/jobs/__tests__/job.service.spec.ts b/utils/jobs/__tests__/job.service.spec.ts index 5d896ea..bf7fd64 100644 --- a/utils/jobs/__tests__/job.service.spec.ts +++ b/utils/jobs/__tests__/job.service.spec.ts @@ -108,6 +108,31 @@ describe('JobService', () => { }); }); + describe('restarts', () => { + it('releaseStale puts running jobs with an old lock back to pending', async () => { + const repo = makeRepo({ update: jest.fn().mockResolvedValue({ affected: 2 }) }); + expect(await new JobService(repo).releaseStale(60_000)).toBe(2); + const [where, set] = repo.update.mock.calls[0]; + expect(where.status).toBe('running'); + expect(where.lockedAt.type).toBe('lessThan'); + expect(Date.now() - where.lockedAt.value.getTime()).toBeGreaterThanOrEqual(60_000); + expect(set).toMatchObject({ status: 'pending', lockedAt: null, lockedBy: null }); + }); + + it('heartbeat renews only the lock this worker holds', async () => { + const repo = makeRepo(); + await new JobService(repo).heartbeat({ id: 3, lockedBy: 'w1' } as Job); + expect(repo.update).toHaveBeenCalledWith({ id: 3, lockedBy: 'w1' }, { lockedAt: expect.any(Date) }); + }); + + it('release returns the job to pending and gives back its attempt', async () => { + const repo = makeRepo(); + const job = { id: 4, status: 'running', attempts: 2, lockedBy: 'w1', lockedAt: new Date() } as Job; + await new JobService(repo).release(job); + expect(repo.save).toHaveBeenCalledWith(expect.objectContaining({ status: 'pending', attempts: 1, lockedBy: null, lockedAt: null })); + }); + }); + describe('markCompleted', () => { it('sets completed state and result', async () => { const repo = makeRepo(); diff --git a/utils/jobs/job-worker.service.ts b/utils/jobs/job-worker.service.ts index 222872b..ad8ec1b 100644 --- a/utils/jobs/job-worker.service.ts +++ b/utils/jobs/job-worker.service.ts @@ -9,6 +9,13 @@ import { JOB_HANDLER, JobHandler, JobResult } from './job.types'; const MAX_IDLE_MS = Number(process.env.JOB_POLL_MS) || 5 * 60_000; const ERROR_RETRY_MS = 30_000; const BATCH = Number(process.env.JOB_WORKER_BATCH) || 5; +// A running job's lock is renewed this often while its handler runs; one not +// renewed for LOCK_TTL_MS belongs to a dead worker and is released to pending. +const LOCK_TTL_MS = Number(process.env.JOB_LOCK_TTL_MS) || 10 * 60_000; +const HEARTBEAT_MS = Math.max(Math.floor(LOCK_TTL_MS / 4), 1000); +// On shutdown, how long to wait for the job in progress. Fits the containers' +// stop_grace_period (40s) after gracefulShutdown's 5s proxy wait. +const SHUTDOWN_WAIT_MS = Number(process.env.JOB_SHUTDOWN_WAIT_MS) || 25_000; /** * Runs due jobs from the jobs table. Instead of polling on a fixed interval it @@ -20,6 +27,12 @@ const BATCH = Number(process.env.JOB_WORKER_BATCH) || 5; * Disabled by default; set WORKER_ENABLED=true on the instance(s) that should * process jobs (typically a dedicated worker process, or the single API * container for small deployments). + * + * Restarts: on shutdown the worker claims nothing more, aborts the running + * handler's signal (JobContext) and waits up to JOB_SHUTDOWN_WAIT_MS; a job + * that stops on the signal goes back to pending for the next instance. A + * worker killed outright stops renewing its locks, and its running jobs are + * released after JOB_LOCK_TTL_MS. */ @Injectable() export class JobWorkerService implements OnApplicationBootstrap, OnModuleDestroy { @@ -31,6 +44,8 @@ export class JobWorkerService implements OnApplicationBootstrap, OnModuleDestroy private stopped = false; private timer: NodeJS.Timeout | null = null; private lastReconcileAt = 0; + private readonly shutdown = new AbortController(); + private polling: Promise | null = null; constructor( @Inject(JOB_HANDLER) handlers: JobHandler[], @@ -55,11 +70,18 @@ export class JobWorkerService implements OnApplicationBootstrap, OnModuleDestroy this.wake(); } - onModuleDestroy(): void { + async onModuleDestroy(): Promise { this.stopped = true; if (this.timer) { clearTimeout(this.timer); } + this.shutdown.abort(); + if (!this.polling) { + return; + } + let timeout: NodeJS.Timeout; + await Promise.race([this.polling, new Promise((resolve) => (timeout = setTimeout(resolve, SHUTDOWN_WAIT_MS)))]); + clearTimeout(timeout); } /** Run due jobs now (or right after the current batch) and re-arm the timer. */ @@ -78,10 +100,15 @@ export class JobWorkerService implements OnApplicationBootstrap, OnModuleDestroy void this.poll(); } - async poll(): Promise { - if (!JobWorkerService.isEnabled() || this.busy) { - return; + poll(): Promise { + if (!JobWorkerService.isEnabled() || this.busy || this.stopped) { + return Promise.resolve(); } + this.polling = this.runPoll().finally(() => (this.polling = null)); + return this.polling; + } + + private async runPoll(): Promise { this.busy = true; let delay = MAX_IDLE_MS; try { @@ -89,11 +116,15 @@ export class JobWorkerService implements OnApplicationBootstrap, OnModuleDestroy this.lastReconcileAt = Date.now(); await this.scheduleSync.reconcile().catch((err) => this.logger.error(`schedule reconcile failed: ${err?.message}`)); } + const released = await this.jobService.releaseStale(LOCK_TTL_MS); + if (released) { + this.logger.warn(`released ${released} stale running job(s) of a stopped worker`); + } let claimed: number; do { this.rerun = false; claimed = await this.processBatch(); - } while (claimed === BATCH || this.rerun); + } while (!this.stopped && (claimed === BATCH || this.rerun)); const next = await this.jobService.nextAvailableAt(); if (next) { delay = Math.min(Math.max(next.getTime() - Date.now(), 0), MAX_IDLE_MS); @@ -116,6 +147,11 @@ export class JobWorkerService implements OnApplicationBootstrap, OnModuleDestroy async processBatch(): Promise { const jobs = await this.jobService.claimNext(this.workerId, BATCH); for (const job of jobs) { + if (this.stopped) { + // Claimed but not started: hand it straight back. + await this.jobService.release(job); + continue; + } await this.runJob(job); } return jobs.length; @@ -137,16 +173,27 @@ export class JobWorkerService implements OnApplicationBootstrap, OnModuleDestroy this.logger.error(`schedule ${scheduleId} chaining failed: ${err?.message}`); } } + const heartbeat = setInterval(() => { + this.jobService.heartbeat(job).catch((err) => this.logger.error(`job ${job.id} heartbeat failed: ${err?.message}`)); + }, HEARTBEAT_MS); try { - const result = (await handler.handle(job)) || null; + const result = (await handler.handle(job, { signal: this.shutdown.signal })) || null; await this.jobService.markCompleted(job, result); await this.notify(job, result); } catch (err) { + if (this.shutdown.signal.aborted) { + // Stopped for the shutdown, not failed: the next instance resumes it. + this.logger.log(`job ${job.id} (${job.type}) interrupted by shutdown, back to pending`); + await this.jobService.release(job); + return; + } this.logger.error(`job ${job.id} (${job.type}) failed: ${err?.message}`); await this.jobService.markFailed(job, err?.message ?? 'unknown error'); if (job.status === 'failed') { await this.notify(job, null); } + } finally { + clearInterval(heartbeat); } } diff --git a/utils/jobs/job.service.ts b/utils/jobs/job.service.ts index f038bee..0283365 100644 --- a/utils/jobs/job.service.ts +++ b/utils/jobs/job.service.ts @@ -1,6 +1,6 @@ import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import { In, LessThanOrEqual, Like, Repository } from 'typeorm'; +import { In, LessThan, LessThanOrEqual, Like, Repository } from 'typeorm'; import { Job, JobStatus } from '@shared/entities/Job.entity'; import { JobResult } from './job.types'; @@ -96,6 +96,40 @@ export class JobService { return claimed; } + /** Renews a running job's lock, so releaseStale leaves it alone. */ + async heartbeat(job: Job): Promise { + job.lockedAt = new Date(); + await this.repo.update({ id: job.id, lockedBy: job.lockedBy }, { lockedAt: job.lockedAt }); + } + + /** + * Running jobs whose lock wasn't renewed for `ttlMs` belong to a worker that + * died (killed mid-deploy, crashed): put them back to pending so a live + * worker resumes them. Returns how many were released. + */ + async releaseStale(ttlMs: number): Promise { + const now = new Date(); + const res = await this.repo.update( + { status: 'running', lockedAt: LessThan(new Date(now.getTime() - ttlMs)) }, + { status: 'pending', lockedAt: null, lockedBy: null, availableAt: now }, + ); + return res.affected ?? 0; + } + + /** + * The worker stopped (shutdown) before or while running the job: back to + * pending at once, and the attempt it took is given back - a deploy is not + * a failure. + */ + async release(job: Job): Promise { + job.status = 'pending'; + job.lockedAt = null; + job.lockedBy = null; + job.availableAt = new Date(); + job.attempts = Math.max(job.attempts - 1, 0); + await this.repo.save(job); + } + async markCompleted(job: Job, result?: JobResult | null): Promise { job.status = 'completed'; job.result = result ?? null; diff --git a/utils/jobs/job.types.ts b/utils/jobs/job.types.ts index bdbd45a..d21d947 100644 --- a/utils/jobs/job.types.ts +++ b/utils/jobs/job.types.ts @@ -15,6 +15,18 @@ export interface JobResult { [key: string]: any; } +/** Passed to a handler alongside its job. */ +export interface JobContext { + /** + * Aborted when the process shuts down (a deploy). A long handler checks it + * between safe steps, saves its progress (e.g. into job.payload) and throws: + * the job then goes back to pending, without using up an attempt, and the + * next instance resumes it. A handler that ignores it simply runs on until + * the shutdown wait (JOB_SHUTDOWN_WAIT_MS) ends. + */ + signal: AbortSignal; +} + /** * A unit of background work. Handlers are Nest providers registered via * JobsModule; they receive their own dependencies through DI. @@ -22,5 +34,5 @@ export interface JobResult { export interface JobHandler { /** Unique key matched against Job.type. */ readonly type: string; - handle(job: Job): Promise; + handle(job: Job, context?: JobContext): Promise; }