Skip to content

Commit 40059cb

Browse files
committed
add spread_fraction rollout
1 parent bc4e47f commit 40059cb

7 files changed

Lines changed: 169 additions & 17 deletions

File tree

apps/webapp/app/env.server.ts

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1684,7 +1684,11 @@ const EnvironmentSchema = z
16841684
SCHEDULE_WORKER_CONCURRENCY_LIMIT: z.coerce.number().int().default(50),
16851685
SCHEDULE_WORKER_SHUTDOWN_TIMEOUT_MS: z.coerce.number().int().default(30_000),
16861686
SCHEDULE_WORKER_DISTRIBUTION_WINDOW_SECONDS: z.coerce.number().int().default(30),
1687-
SCHEDULE_WORKER_CRON_SPREAD_ENABLED: BoolEnv.default(false),
1687+
SCHEDULE_WORKER_CRON_SPREAD_FRACTION: z.coerce
1688+
.number()
1689+
.catch(0)
1690+
.default(0)
1691+
.transform((value) => (Number.isFinite(value) ? Math.min(1, Math.max(0, value)) : 0)),
16881692

16891693
SCHEDULE_WORKER_REDIS_HOST: z
16901694
.string()

apps/webapp/app/v3/scheduleEngine.server.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ function createScheduleEngine() {
7373
seconds: env.SCHEDULE_WORKER_DISTRIBUTION_WINDOW_SECONDS,
7474
},
7575
schedulePhaseSecret: env.ENCRYPTION_KEY,
76-
cronSpreadEnabled: env.SCHEDULE_WORKER_CRON_SPREAD_ENABLED,
76+
cronSpreadFraction: env.SCHEDULE_WORKER_CRON_SPREAD_FRACTION,
7777
tracer,
7878
meter,
7979
onTriggerScheduledTask: async ({

internal-packages/schedule-engine/src/engine/index.ts

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,11 @@ import type {
1515
TriggerScheduledTaskCallback,
1616
TriggerScheduleParams,
1717
} from "./types.js";
18-
import { calculateSchedulePhase, type NormalizedScheduleWindow } from "./scheduleTiming.js";
18+
import {
19+
calculateSchedulePhase,
20+
SCHEDULE_PHASE_DENOMINATOR,
21+
type NormalizedScheduleWindow,
22+
} from "./scheduleTiming.js";
1923
import { scheduleWorkerCatalog } from "./workerCatalog.js";
2024
import { tryCatch } from "@trigger.dev/core/utils";
2125

@@ -208,6 +212,8 @@ export class ScheduleEngine {
208212
const fromTimestamp = params.fromTimestamp ?? registrationTime;
209213
span.setAttribute("from_timestamp", fromTimestamp.toISOString());
210214

215+
const cronSpreadActive = this.#isCronSpreadActive(schedulePhase);
216+
211217
const {
212218
nominalAt,
213219
candidateEffectiveAt,
@@ -225,11 +231,12 @@ export class ScheduleEngine {
225231
now: registrationTime,
226232
schedulePhase,
227233
window: scheduleWindow,
228-
cronSpreadEnabled: this.options.cronSpreadEnabled,
234+
cronSpreadEnabled: cronSpreadActive,
229235
});
230236
const appliedDelayMs = effectiveAt.getTime() - nominalAt.getTime();
231237

232-
span.setAttribute("cron_spread_enabled", this.options.cronSpreadEnabled);
238+
span.setAttribute("cron_spread_fraction", this.options.cronSpreadFraction);
239+
span.setAttribute("cron_spread_active", cronSpreadActive);
233240
span.setAttribute("schedule_window_type", scheduleWindow?.type ?? "none");
234241
span.setAttribute("next_scheduled_timestamp", nominalAt.toISOString());
235242
span.setAttribute("candidate_effective_schedule_time", candidateEffectiveAt.toISOString());
@@ -268,7 +275,7 @@ export class ScheduleEngine {
268275
nominalAt: nominalAt.toISOString(),
269276
candidateEffectiveAt: candidateEffectiveAt.toISOString(),
270277
effectiveAt: effectiveAt.toISOString(),
271-
cronSpreadEnabled: this.options.cronSpreadEnabled,
278+
cronSpreadActive,
272279
scheduleWindowType: scheduleWindow?.type ?? "none",
273280
candidateDelayMs,
274281
appliedDelayMs,
@@ -528,14 +535,16 @@ export class ScheduleEngine {
528535
environmentId: instance.environmentId,
529536
deduplicationKey: instance.taskSchedule.deduplicationKey,
530537
});
538+
const cronSpreadActive = this.#isCronSpreadActive(schedulePhase);
539+
span.setAttribute("cron_spread_active", cronSpreadActive);
531540
const nextOccurrence = calculateNextSchedulableOccurrence({
532541
schedule: instance.taskSchedule.generatorExpression,
533542
timezone: instance.taskSchedule.timezone,
534543
afterNominal: exactScheduleTime,
535544
now: actualExecutionTime,
536545
schedulePhase,
537546
window: scheduleWindow,
538-
cronSpreadEnabled: this.options.cronSpreadEnabled,
547+
cronSpreadEnabled: cronSpreadActive,
539548
});
540549
const upcoming = [
541550
nextOccurrence.nominalAt,
@@ -757,6 +766,16 @@ export class ScheduleEngine {
757766
});
758767
}
759768

769+
/**
770+
* Per-schedule rollout gate for cron spread. The schedule's deterministic
771+
* phase doubles as a stable sampling key: raising the fraction is strictly
772+
* additive (a schedule never leaves the rollout once included), and 0/1 map
773+
* to fully off/on.
774+
*/
775+
#isCronSpreadActive(schedulePhase: number): boolean {
776+
return schedulePhase < this.options.cronSpreadFraction * SCHEDULE_PHASE_DENOMINATOR;
777+
}
778+
760779
/**
761780
* Enqueues a scheduled task with distributed execution timing
762781
*/

internal-packages/schedule-engine/src/engine/types.ts

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,13 @@ export interface ScheduleEngineOptions {
5252
seconds: number;
5353
};
5454
schedulePhaseSecret: string | Buffer;
55-
cronSpreadEnabled: boolean;
55+
/**
56+
* Fraction of schedules (0 to 1) with cron spread active, gated on each
57+
* schedule's deterministic phase. 0 disables spreading entirely; 1 enables
58+
* it for every schedule. Raising the fraction is strictly additive — phases
59+
* are stable, so a schedule never leaves the rollout once included.
60+
*/
61+
cronSpreadFraction: number;
5662
tracer?: Tracer;
5763
meter?: Meter;
5864
onTriggerScheduledTask: TriggerScheduledTaskCallback;

internal-packages/schedule-engine/test/scheduleEngine.test.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ describe("ScheduleEngine Integration", () => {
2323
redis: redisOptions,
2424
distributionWindow: { seconds: 10 },
2525
schedulePhaseSecret: "test-schedule-phase-secret",
26-
cronSpreadEnabled: true,
26+
cronSpreadFraction: 1,
2727
worker: {
2828
concurrency: 1,
2929
disabled: false, // Enable worker for full integration test

internal-packages/schedule-engine/test/scheduleEngine2.test.ts

Lines changed: 125 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import {
66
calculateEffectiveScheduleTime,
77
calculateNextNominalTimestamp,
88
calculateSchedulePhase,
9+
SCHEDULE_PHASE_DENOMINATOR,
910
ScheduleEngine,
1011
} from "../src/index.js";
1112
import { calculateDistributedExecutionTime } from "../src/engine/distributedScheduling.js";
@@ -27,7 +28,7 @@ describe("ScheduleEngine Integration (part 2)", () => {
2728
redis: redisOptions,
2829
distributionWindow: { seconds: 10 },
2930
schedulePhaseSecret: "test-schedule-phase-secret",
30-
cronSpreadEnabled: false,
31+
cronSpreadFraction: 0,
3132
worker: {
3233
concurrency: 1,
3334
disabled: true, // Don't actually run the worker — calling triggerScheduledTask directly
@@ -169,7 +170,7 @@ describe("ScheduleEngine Integration (part 2)", () => {
169170
redis: redisOptions,
170171
distributionWindow: { seconds: 10 },
171172
schedulePhaseSecret,
172-
cronSpreadEnabled: true,
173+
cronSpreadFraction: 1,
173174
worker: {
174175
concurrency: 1,
175176
disabled: true,
@@ -366,4 +367,126 @@ describe("ScheduleEngine Integration (part 2)", () => {
366367
}
367368
}
368369
);
370+
371+
containerTest(
372+
"gates cron spread per schedule via the rollout fraction",
373+
{ timeout: 30_000 },
374+
async ({ prisma, redisOptions }) => {
375+
const schedulePhaseSecret = "test-schedule-phase-secret";
376+
377+
const organization = await prisma.organization.create({
378+
data: { title: "Spread Fraction Org", slug: "spread-fraction-org" },
379+
});
380+
const project = await prisma.project.create({
381+
data: {
382+
name: "Spread Fraction Project",
383+
slug: "spread-fraction-project",
384+
externalRef: "spread-fraction-ref",
385+
organizationId: organization.id,
386+
},
387+
});
388+
const environment = await prisma.runtimeEnvironment.create({
389+
data: {
390+
slug: "spread-fraction-env",
391+
type: "PRODUCTION",
392+
projectId: project.id,
393+
organizationId: organization.id,
394+
apiKey: "tr_spread_fraction",
395+
pkApiKey: "pk_spread_fraction",
396+
shortcode: "spread",
397+
},
398+
});
399+
const taskSchedule = await prisma.taskSchedule.create({
400+
data: {
401+
friendlyId: "sched_spread_fraction",
402+
taskIdentifier: "spread-fraction-task",
403+
projectId: project.id,
404+
deduplicationKey: "spread-fraction-dedup",
405+
generatorExpression: "*/5 * * * *",
406+
generatorDescription: "Every 5 minutes",
407+
timezone: "UTC",
408+
type: "DECLARATIVE",
409+
},
410+
});
411+
const scheduleInstance = await prisma.taskScheduleInstance.create({
412+
data: {
413+
taskScheduleId: taskSchedule.id,
414+
environmentId: environment.id,
415+
projectId: project.id,
416+
},
417+
});
418+
419+
const phase = calculateSchedulePhase({
420+
secret: schedulePhaseSecret,
421+
environmentId: environment.id,
422+
deduplicationKey: taskSchedule.deduplicationKey,
423+
});
424+
425+
// The gate is `phase < fraction * DENOMINATOR`. Dividing and multiplying
426+
// by 2^31 is exact in floating point, so `phase / DENOMINATOR` excludes
427+
// this schedule and `(phase + 1) / DENOMINATOR` includes it.
428+
const excludingFraction = phase / SCHEDULE_PHASE_DENOMINATOR;
429+
const includingFraction = (phase + 1) / SCHEDULE_PHASE_DENOMINATOR;
430+
431+
const createEngine = (cronSpreadFraction: number) =>
432+
new ScheduleEngine({
433+
prisma,
434+
redis: redisOptions,
435+
distributionWindow: { seconds: 10 },
436+
schedulePhaseSecret,
437+
cronSpreadFraction,
438+
worker: {
439+
concurrency: 1,
440+
disabled: true,
441+
pollIntervalMs: 1000,
442+
},
443+
tracer: trace.getTracer("test", "0.0.0"),
444+
onTriggerScheduledTask: async () => ({ success: true }),
445+
isDevEnvironmentConnectedHandler: vi.fn().mockResolvedValue(true),
446+
});
447+
448+
const jobId = `scheduled-task-instance:${scheduleInstance.id}`;
449+
450+
const excludedEngine = createEngine(excludingFraction);
451+
try {
452+
await excludedEngine.registerNextTaskScheduleInstance({ instanceId: scheduleInstance.id });
453+
const job = await excludedEngine.getJob(jobId);
454+
const payload = job!.item as unknown as {
455+
exactScheduleTime: string;
456+
effectiveScheduleTime: string;
457+
};
458+
// Spread inactive: the effective time is the nominal tick.
459+
expect(new Date(payload.effectiveScheduleTime)).toEqual(
460+
new Date(payload.exactScheduleTime)
461+
);
462+
} finally {
463+
await excludedEngine.quit();
464+
}
465+
466+
const includedEngine = createEngine(includingFraction);
467+
try {
468+
await includedEngine.registerNextTaskScheduleInstance({ instanceId: scheduleInstance.id });
469+
const job = await includedEngine.getJob(jobId);
470+
const payload = job!.item as unknown as {
471+
exactScheduleTime: string;
472+
effectiveScheduleTime: string;
473+
};
474+
const nominalAt = new Date(payload.exactScheduleTime);
475+
const nextNominalAt = calculateNextNominalTimestamp(
476+
taskSchedule.generatorExpression,
477+
taskSchedule.timezone,
478+
nominalAt
479+
);
480+
// Spread active with no window configured: the 60s baseline applies.
481+
const { effectiveAt } = calculateEffectiveScheduleTime({
482+
nominalAt,
483+
nextNominalAt,
484+
schedulePhase: phase,
485+
});
486+
expect(new Date(payload.effectiveScheduleTime)).toEqual(effectiveAt);
487+
} finally {
488+
await includedEngine.quit();
489+
}
490+
}
491+
);
369492
});

internal-packages/schedule-engine/test/scheduleRecovery.test.ts

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ describe("Schedule Recovery", () => {
1717
redis: redisOptions,
1818
distributionWindow: { seconds: 10 },
1919
schedulePhaseSecret: "test-schedule-phase-secret",
20-
cronSpreadEnabled: true,
20+
cronSpreadFraction: 1,
2121
worker: {
2222
concurrency: 1,
2323
disabled: true, // Disable worker to prevent automatic execution
@@ -121,7 +121,7 @@ describe("Schedule Recovery", () => {
121121
redis: redisOptions,
122122
distributionWindow: { seconds: 10 },
123123
schedulePhaseSecret: "test-schedule-phase-secret",
124-
cronSpreadEnabled: true,
124+
cronSpreadFraction: 1,
125125
worker: {
126126
concurrency: 1,
127127
disabled: true, // Disable worker to prevent automatic execution
@@ -228,7 +228,7 @@ describe("Schedule Recovery", () => {
228228
redis: redisOptions,
229229
distributionWindow: { seconds: 10 },
230230
schedulePhaseSecret: "test-schedule-phase-secret",
231-
cronSpreadEnabled: true,
231+
cronSpreadFraction: 1,
232232
worker: {
233233
concurrency: 1,
234234
disabled: true, // Disable worker to prevent automatic execution
@@ -341,7 +341,7 @@ describe("Schedule Recovery", () => {
341341
redis: redisOptions,
342342
distributionWindow: { seconds: 10 },
343343
schedulePhaseSecret: "test-schedule-phase-secret",
344-
cronSpreadEnabled: true,
344+
cronSpreadFraction: 1,
345345
worker: {
346346
concurrency: 1,
347347
disabled: true, // Disable worker to prevent automatic execution
@@ -413,7 +413,7 @@ describe("Schedule Recovery", () => {
413413
redis: redisOptions,
414414
distributionWindow: { seconds: 10 },
415415
schedulePhaseSecret: "test-schedule-phase-secret",
416-
cronSpreadEnabled: true,
416+
cronSpreadFraction: 1,
417417
worker: { concurrency: 1, disabled: true, pollIntervalMs: 1000 },
418418
tracer: trace.getTracer("test", "0.0.0"),
419419
onTriggerScheduledTask: async () => ({ success: true }),
@@ -516,7 +516,7 @@ describe("Schedule Recovery", () => {
516516
redis: redisOptions,
517517
distributionWindow: { seconds: 10 },
518518
schedulePhaseSecret: "test-schedule-phase-secret",
519-
cronSpreadEnabled: true,
519+
cronSpreadFraction: 1,
520520
worker: { concurrency: 1, disabled: true, pollIntervalMs: 1000 },
521521
tracer: trace.getTracer("test", "0.0.0"),
522522
onTriggerScheduledTask: async () => ({ success: true }),

0 commit comments

Comments
 (0)