Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 79 additions & 1 deletion utils/jobs/__tests__/job-worker.service.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ describe('JobWorkerService', () => {
markFailed: jest.fn(),
nextAvailableAt: jest.fn().mockResolvedValue(null),
onEnqueue: jest.fn(),
releaseStale: jest.fn().mockResolvedValue(0),
release: jest.fn(),
heartbeat: jest.fn().mockResolvedValue(undefined),
} as any;
}

Expand All @@ -33,7 +36,7 @@ describe('JobWorkerService', () => {
worker = new JobWorkerService([handler], jobService, makeScheduleSync());
const job = { id: 1, type: 'demo', payload: {} } as Job;
await worker.runJob(job);
expect(handler.handle).toHaveBeenCalledWith(job);
expect(handler.handle).toHaveBeenCalledWith(job, { signal: expect.any(AbortSignal) });
expect(jobService.markCompleted).toHaveBeenCalledWith(job, { summary: 'ok' });
});

Expand Down Expand Up @@ -115,6 +118,81 @@ describe('JobWorkerService', () => {
expect(jobService.markCompleted).toHaveBeenCalledWith(job, null);
});

describe('restarts', () => {
it('releases stale running jobs of a dead worker before claiming', async () => {
process.env.WORKER_ENABLED = 'true';
const jobService = makeJobService();
worker = new JobWorkerService([], jobService, makeScheduleSync());
await worker.poll();
expect(jobService.releaseStale).toHaveBeenCalledWith(10 * 60_000);
expect(jobService.releaseStale.mock.invocationCallOrder[0]).toBeLessThan(jobService.claimNext.mock.invocationCallOrder[0]);
});

it('on shutdown aborts the handler, waits for it and puts its job back to pending', async () => {
process.env.WORKER_ENABLED = 'true';
let started: () => void;
const running = new Promise<void>((resolve) => (started = resolve));
const handler: JobHandler = {
type: 'demo',
handle: jest.fn((_job, { signal }) => {
started();
return new Promise((_resolve, reject) => signal.addEventListener('abort', () => setTimeout(() => reject(new Error('stopped')), 10)));
}),
};
const job = { id: 10, type: 'demo', payload: {}, attempts: 1 } as Job;
const jobService = makeJobService();
jobService.claimNext.mockResolvedValue([job]);
worker = new JobWorkerService([handler], jobService, makeScheduleSync());
void worker.poll();
await running;
await worker.onModuleDestroy();
expect(jobService.release).toHaveBeenCalledWith(job);
expect(jobService.markFailed).not.toHaveBeenCalled();
// Stopped: no more polls.
await worker.poll();
expect(jobService.claimNext).toHaveBeenCalledTimes(1);
});

it('a job that finishes during the shutdown wait is completed', async () => {
process.env.WORKER_ENABLED = 'true';
const handler: JobHandler = { type: 'demo', handle: jest.fn(() => new Promise((resolve) => setTimeout(() => resolve({ summary: 'ok' }), 20))) };
const job = { id: 11, type: 'demo', payload: {} } as Job;
const jobService = makeJobService();
jobService.claimNext.mockResolvedValue([job]);
worker = new JobWorkerService([handler], jobService, makeScheduleSync());
void worker.poll();
await new Promise((r) => setTimeout(r, 5));
await worker.onModuleDestroy();
expect(jobService.markCompleted).toHaveBeenCalledWith(job, { summary: 'ok' });
expect(jobService.release).not.toHaveBeenCalled();
});

it('hands back jobs claimed but not started when stopping', async () => {
const jobs = [{ id: 12, type: 'demo' }, { id: 13, type: 'demo' }] as Job[];
const jobService = makeJobService();
jobService.claimNext.mockResolvedValue(jobs);
let worker2: JobWorkerService;
const handler: JobHandler = { type: 'demo', handle: jest.fn(async () => { void worker2.onModuleDestroy(); }) };
worker2 = worker = new JobWorkerService([handler], jobService, makeScheduleSync());
await worker.processBatch();
expect(handler.handle).toHaveBeenCalledTimes(1);
expect(jobService.markCompleted).toHaveBeenCalledWith(jobs[0], null);
expect(jobService.release).toHaveBeenCalledWith(jobs[1]);
});

it('renews the lock while the handler runs', async () => {
jest.useFakeTimers();
const handler: JobHandler = { type: 'demo', handle: jest.fn(() => new Promise((resolve) => setTimeout(resolve, 6 * 60_000))) };
const jobService = makeJobService();
worker = new JobWorkerService([handler], jobService, makeScheduleSync());
const run = worker.runJob({ id: 14, type: 'demo', payload: {} } as Job);
await jest.advanceTimersByTimeAsync(6 * 60_000);
await run;
expect(jobService.heartbeat).toHaveBeenCalledTimes(2);
jest.useRealTimers();
});
});

describe('timer', () => {
beforeEach(() => {
jest.useFakeTimers();
Expand Down
25 changes: 25 additions & 0 deletions utils/jobs/__tests__/job.service.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,31 @@ describe('JobService', () => {
});
});

describe('restarts', () => {
it('releaseStale puts running jobs with an old lock back to pending', async () => {
const repo = makeRepo({ update: jest.fn().mockResolvedValue({ affected: 2 }) });
expect(await new JobService(repo).releaseStale(60_000)).toBe(2);
const [where, set] = repo.update.mock.calls[0];
expect(where.status).toBe('running');
expect(where.lockedAt.type).toBe('lessThan');
expect(Date.now() - where.lockedAt.value.getTime()).toBeGreaterThanOrEqual(60_000);
expect(set).toMatchObject({ status: 'pending', lockedAt: null, lockedBy: null });
});

it('heartbeat renews only the lock this worker holds', async () => {
const repo = makeRepo();
await new JobService(repo).heartbeat({ id: 3, lockedBy: 'w1' } as Job);
expect(repo.update).toHaveBeenCalledWith({ id: 3, lockedBy: 'w1' }, { lockedAt: expect.any(Date) });
});

it('release returns the job to pending and gives back its attempt', async () => {
const repo = makeRepo();
const job = { id: 4, status: 'running', attempts: 2, lockedBy: 'w1', lockedAt: new Date() } as Job;
await new JobService(repo).release(job);
expect(repo.save).toHaveBeenCalledWith(expect.objectContaining({ status: 'pending', attempts: 1, lockedBy: null, lockedAt: null }));
});
});

describe('markCompleted', () => {
it('sets completed state and result', async () => {
const repo = makeRepo();
Expand Down
59 changes: 53 additions & 6 deletions utils/jobs/job-worker.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,13 @@ import { JOB_HANDLER, JobHandler, JobResult } from './job.types';
const MAX_IDLE_MS = Number(process.env.JOB_POLL_MS) || 5 * 60_000;
const ERROR_RETRY_MS = 30_000;
const BATCH = Number(process.env.JOB_WORKER_BATCH) || 5;
// A running job's lock is renewed this often while its handler runs; one not
// renewed for LOCK_TTL_MS belongs to a dead worker and is released to pending.
const LOCK_TTL_MS = Number(process.env.JOB_LOCK_TTL_MS) || 10 * 60_000;
const HEARTBEAT_MS = Math.max(Math.floor(LOCK_TTL_MS / 4), 1000);
// On shutdown, how long to wait for the job in progress. Fits the containers'
// stop_grace_period (40s) after gracefulShutdown's 5s proxy wait.
const SHUTDOWN_WAIT_MS = Number(process.env.JOB_SHUTDOWN_WAIT_MS) || 25_000;

/**
* Runs due jobs from the jobs table. Instead of polling on a fixed interval it
Expand All @@ -20,6 +27,12 @@ const BATCH = Number(process.env.JOB_WORKER_BATCH) || 5;
* Disabled by default; set WORKER_ENABLED=true on the instance(s) that should
* process jobs (typically a dedicated worker process, or the single API
* container for small deployments).
*
* Restarts: on shutdown the worker claims nothing more, aborts the running
* handler's signal (JobContext) and waits up to JOB_SHUTDOWN_WAIT_MS; a job
* that stops on the signal goes back to pending for the next instance. A
* worker killed outright stops renewing its locks, and its running jobs are
* released after JOB_LOCK_TTL_MS.
*/
@Injectable()
export class JobWorkerService implements OnApplicationBootstrap, OnModuleDestroy {
Expand All @@ -31,6 +44,8 @@ export class JobWorkerService implements OnApplicationBootstrap, OnModuleDestroy
private stopped = false;
private timer: NodeJS.Timeout | null = null;
private lastReconcileAt = 0;
private readonly shutdown = new AbortController();
private polling: Promise<void> | null = null;

constructor(
@Inject(JOB_HANDLER) handlers: JobHandler[],
Expand All @@ -55,11 +70,18 @@ export class JobWorkerService implements OnApplicationBootstrap, OnModuleDestroy
this.wake();
}

onModuleDestroy(): void {
async onModuleDestroy(): Promise<void> {
this.stopped = true;
if (this.timer) {
clearTimeout(this.timer);
}
this.shutdown.abort();
if (!this.polling) {
return;
}
let timeout: NodeJS.Timeout;
await Promise.race([this.polling, new Promise((resolve) => (timeout = setTimeout(resolve, SHUTDOWN_WAIT_MS)))]);
clearTimeout(timeout);
}

/** Run due jobs now (or right after the current batch) and re-arm the timer. */
Expand All @@ -78,22 +100,31 @@ export class JobWorkerService implements OnApplicationBootstrap, OnModuleDestroy
void this.poll();
}

async poll(): Promise<void> {
if (!JobWorkerService.isEnabled() || this.busy) {
return;
poll(): Promise<void> {
if (!JobWorkerService.isEnabled() || this.busy || this.stopped) {
return Promise.resolve();
}
this.polling = this.runPoll().finally(() => (this.polling = null));
return this.polling;
}

private async runPoll(): Promise<void> {
this.busy = true;
let delay = MAX_IDLE_MS;
try {
if (Date.now() - this.lastReconcileAt >= MAX_IDLE_MS) {
this.lastReconcileAt = Date.now();
await this.scheduleSync.reconcile().catch((err) => this.logger.error(`schedule reconcile failed: ${err?.message}`));
}
const released = await this.jobService.releaseStale(LOCK_TTL_MS);
if (released) {
this.logger.warn(`released ${released} stale running job(s) of a stopped worker`);
}
let claimed: number;
do {
this.rerun = false;
claimed = await this.processBatch();
} while (claimed === BATCH || this.rerun);
} while (!this.stopped && (claimed === BATCH || this.rerun));
const next = await this.jobService.nextAvailableAt();
if (next) {
delay = Math.min(Math.max(next.getTime() - Date.now(), 0), MAX_IDLE_MS);
Expand All @@ -116,6 +147,11 @@ export class JobWorkerService implements OnApplicationBootstrap, OnModuleDestroy
async processBatch(): Promise<number> {
const jobs = await this.jobService.claimNext(this.workerId, BATCH);
for (const job of jobs) {
if (this.stopped) {
// Claimed but not started: hand it straight back.
await this.jobService.release(job);
continue;
}
await this.runJob(job);
}
return jobs.length;
Expand All @@ -137,16 +173,27 @@ export class JobWorkerService implements OnApplicationBootstrap, OnModuleDestroy
this.logger.error(`schedule ${scheduleId} chaining failed: ${err?.message}`);
}
}
const heartbeat = setInterval(() => {
this.jobService.heartbeat(job).catch((err) => this.logger.error(`job ${job.id} heartbeat failed: ${err?.message}`));
}, HEARTBEAT_MS);
try {
const result = (await handler.handle(job)) || null;
const result = (await handler.handle(job, { signal: this.shutdown.signal })) || null;
await this.jobService.markCompleted(job, result);
await this.notify(job, result);
} catch (err) {
if (this.shutdown.signal.aborted) {
// Stopped for the shutdown, not failed: the next instance resumes it.
this.logger.log(`job ${job.id} (${job.type}) interrupted by shutdown, back to pending`);
await this.jobService.release(job);
return;
}
this.logger.error(`job ${job.id} (${job.type}) failed: ${err?.message}`);
await this.jobService.markFailed(job, err?.message ?? 'unknown error');
if (job.status === 'failed') {
await this.notify(job, null);
}
} finally {
clearInterval(heartbeat);
}
}

Expand Down
36 changes: 35 additions & 1 deletion utils/jobs/job.service.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { In, LessThanOrEqual, Like, Repository } from 'typeorm';
import { In, LessThan, LessThanOrEqual, Like, Repository } from 'typeorm';
import { Job, JobStatus } from '@shared/entities/Job.entity';
import { JobResult } from './job.types';

Expand Down Expand Up @@ -96,6 +96,40 @@ export class JobService {
return claimed;
}

/** Renews a running job's lock, so releaseStale leaves it alone. */
async heartbeat(job: Job): Promise<void> {
job.lockedAt = new Date();
await this.repo.update({ id: job.id, lockedBy: job.lockedBy }, { lockedAt: job.lockedAt });
}

/**
* Running jobs whose lock wasn't renewed for `ttlMs` belong to a worker that
* died (killed mid-deploy, crashed): put them back to pending so a live
* worker resumes them. Returns how many were released.
*/
async releaseStale(ttlMs: number): Promise<number> {
const now = new Date();
const res = await this.repo.update(
{ status: 'running', lockedAt: LessThan(new Date(now.getTime() - ttlMs)) },
{ status: 'pending', lockedAt: null, lockedBy: null, availableAt: now },
);
return res.affected ?? 0;
}

/**
* The worker stopped (shutdown) before or while running the job: back to
* pending at once, and the attempt it took is given back - a deploy is not
* a failure.
*/
async release(job: Job): Promise<void> {
job.status = 'pending';
job.lockedAt = null;
job.lockedBy = null;
job.availableAt = new Date();
job.attempts = Math.max(job.attempts - 1, 0);
await this.repo.save(job);
}

async markCompleted(job: Job, result?: JobResult | null): Promise<void> {
job.status = 'completed';
job.result = result ?? null;
Expand Down
14 changes: 13 additions & 1 deletion utils/jobs/job.types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,24 @@ export interface JobResult {
[key: string]: any;
}

/** Passed to a handler alongside its job. */
export interface JobContext {
/**
* Aborted when the process shuts down (a deploy). A long handler checks it
* between safe steps, saves its progress (e.g. into job.payload) and throws:
* the job then goes back to pending, without using up an attempt, and the
* next instance resumes it. A handler that ignores it simply runs on until
* the shutdown wait (JOB_SHUTDOWN_WAIT_MS) ends.
*/
signal: AbortSignal;
}

/**
* A unit of background work. Handlers are Nest providers registered via
* JobsModule; they receive their own dependencies through DI.
*/
export interface JobHandler {
/** Unique key matched against Job.type. */
readonly type: string;
handle(job: Job): Promise<JobResult | void>;
handle(job: Job, context?: JobContext): Promise<JobResult | void>;
}
Loading