From dc20f17ffbd4184da1850d1fab0f61fc9d1c14f1 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Sat, 1 Aug 2026 07:55:06 +0100 Subject: [PATCH 1/2] perf(webapp,run-engine,database): resolve the newest worker and deployment by createdAt Ordering these lookups by id makes Postgres scan the primary key backwards betting on an early match for the target environment. When that bet loses the scan crosses a large part of the index. Ordering by createdAt then id keeps the lookup on a composite index instead. Adds (environmentId, createdAt) on WorkerDeployment, which had no matching index. BackgroundWorker already had one. --- .../worker-deployment-lookup-ordering.md | 6 + .../clickhouseRunsRepository.server.ts | 3 - .../app/v3/models/workerDeployment.server.ts | 4 +- .../controlPlaneResolver.server.ts | 5 +- .../controlPlaneResolver.server.test.ts | 111 +++++++++++++- .../migration.sql | 1 + .../database/prisma/schema.prisma | 1 + .../src/engine/controlPlaneResolver.ts | 11 +- ...dequeueSystem.controlPlaneResolver.test.ts | 139 ++++++++++++++++++ 9 files changed, 261 insertions(+), 20 deletions(-) create mode 100644 .server-changes/worker-deployment-lookup-ordering.md create mode 100644 internal-packages/database/prisma/migrations/20260731160000_add_worker_deployment_environment_id_created_at_index/migration.sql diff --git a/.server-changes/worker-deployment-lookup-ordering.md b/.server-changes/worker-deployment-lookup-ordering.md new file mode 100644 index 0000000000..1dbfbdd253 --- /dev/null +++ b/.server-changes/worker-deployment-lookup-ordering.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +Speeds up resolving the latest worker version and deployment for an environment, removing an occasional stall when triggering runs in projects that have accumulated many deployed versions. diff --git a/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts b/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts index 2aa9d5f3a6..c9fefd1da1 100644 --- a/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts +++ b/apps/webapp/app/services/runsRepository/clickhouseRunsRepository.server.ts @@ -271,9 +271,6 @@ export class ClickHouseRunsRepository implements IRunsRepository { in: ids, }, }, - orderBy: { - id: "desc", - }, select: { id: true, friendlyId: true, diff --git a/apps/webapp/app/v3/models/workerDeployment.server.ts b/apps/webapp/app/v3/models/workerDeployment.server.ts index 5934cd0762..62d7ea6313 100644 --- a/apps/webapp/app/v3/models/workerDeployment.server.ts +++ b/apps/webapp/app/v3/models/workerDeployment.server.ts @@ -133,9 +133,7 @@ export async function findCurrentWorkerDeployment({ environmentId, type, }, - orderBy: { - id: "desc", - }, + orderBy: [{ createdAt: "desc" }, { id: "desc" }], select: { id: true, imageReference: true, diff --git a/apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.ts b/apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.ts index d6a706c966..eb19a7fb6c 100644 --- a/apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.ts +++ b/apps/webapp/app/v3/runOpsMigration/controlPlaneResolver.server.ts @@ -432,7 +432,7 @@ export class ControlPlaneResolver { // MANAGED deployment. const latestV2Deployment = await client.workerDeployment.findFirst({ where: { environmentId, type: "MANAGED" }, - orderBy: { id: "desc" }, + orderBy: [{ createdAt: "desc" }, { id: "desc" }], include: { worker: { include: { tasks: true, queues: true } } }, }); @@ -455,7 +455,6 @@ export class ControlPlaneResolver { const worker = await client.backgroundWorker.findFirst({ where: { id: workerId }, include: { deployment: true, tasks: true, queues: true }, - orderBy: { id: "desc" }, }); if (!worker) { @@ -472,7 +471,7 @@ export class ControlPlaneResolver { const worker = await client.backgroundWorker.findFirst({ where: { runtimeEnvironmentId: environmentId }, include: { tasks: true, queues: true }, - orderBy: { id: "desc" }, + orderBy: [{ createdAt: "desc" }, { id: "desc" }], }); if (!worker) { diff --git a/apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts b/apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts index e85843114d..af734ca1f7 100644 --- a/apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts +++ b/apps/webapp/test/v3/runOpsMigration/controlPlaneResolver.server.test.ts @@ -62,7 +62,12 @@ async function seedControlPlane(prisma: PrismaClient) { async function seedWorker( prisma: PrismaClient, ctx: { projectId: string; environmentId: string }, - opts?: { promote?: boolean } + opts?: { + promote?: boolean; + createdAt?: Date; + deploymentType?: "MANAGED" | "UNMANAGED" | "V1"; + deploymentCreatedAt?: Date; + } ) { const n = seedCounter++; const worker = await prisma.backgroundWorker.create({ @@ -74,6 +79,7 @@ async function seedWorker( version: `2024.1.${n}`, metadata: {}, engine: "V2", + ...(opts?.createdAt ? { createdAt: opts.createdAt } : {}), }, }); const task = await prisma.backgroundWorkerTask.create({ @@ -104,11 +110,12 @@ async function seedWorker( contentHash: `hash_${n}`, version: worker.version, shortCode: `dep_${n}`, - type: "MANAGED", + type: opts?.deploymentType ?? "MANAGED", status: "DEPLOYED", projectId: ctx.projectId, environmentId: ctx.environmentId, workerId: worker.id, + ...(opts?.deploymentCreatedAt ? { createdAt: opts.deploymentCreatedAt } : {}), }, }); await prisma.workerDeploymentPromotion.create({ @@ -748,3 +755,103 @@ heteroPostgresTest( expect(reads()).toBe(readsAfterFirst * 2); } ); + +heteroPostgresTest( + "resolveWorkerVersion (DEVELOPMENT) resolves the newest worker by createdAt, not by id", + async ({ prisma14 }) => { + const { environment, project } = await seedControlPlane(prisma14); + const ctx = { projectId: project.id, environmentId: environment.id }; + + const newest = await seedWorker(prisma14, ctx, { + createdAt: new Date("2026-07-31T12:00:00.000Z"), + }); + const oldest = await seedWorker(prisma14, ctx, { + createdAt: new Date("2026-07-30T12:00:00.000Z"), + }); + + expect(oldest.worker.id > newest.worker.id).toBe(true); + expect(oldest.worker.createdAt < newest.worker.createdAt).toBe(true); + + const resolver = new ControlPlaneResolver({ + controlPlaneReplica: prisma14, + controlPlanePrimary: prisma14, + cache: new ControlPlaneCache(), + splitEnabled: () => true, + }); + + const resolved = await resolver.resolveWorkerVersion({ + environmentId: environment.id, + type: "DEVELOPMENT", + }); + + expect(resolved).not.toBeNull(); + expect(resolved!.worker.id).toBe(newest.worker.id); + } +); + +heteroPostgresTest( + "resolveWorkerVersion latest-MANAGED fallback resolves by createdAt, not by id", + async ({ prisma14 }) => { + const { environment, project } = await seedControlPlane(prisma14); + const ctx = { projectId: project.id, environmentId: environment.id }; + + await seedWorker(prisma14, ctx, { promote: true, deploymentType: "V1" }); + + const newest = await seedWorker(prisma14, ctx, { + promote: false, + deploymentCreatedAt: new Date("2026-07-31T12:00:00.000Z"), + }); + const oldest = await seedWorker(prisma14, ctx, { + promote: false, + deploymentCreatedAt: new Date("2026-07-30T12:00:00.000Z"), + }); + + const newestDeployment = await prisma14.workerDeployment.create({ + data: { + friendlyId: `deployment_newest_${environment.id}`, + contentHash: "hash_newest", + version: newest.worker.version, + shortCode: "dep_newest", + type: "MANAGED", + status: "DEPLOYED", + projectId: project.id, + environmentId: environment.id, + workerId: newest.worker.id, + createdAt: new Date("2026-07-31T12:00:00.000Z"), + }, + }); + const oldestDeployment = await prisma14.workerDeployment.create({ + data: { + friendlyId: `deployment_oldest_${environment.id}`, + contentHash: "hash_oldest", + version: oldest.worker.version, + shortCode: "dep_oldest", + type: "MANAGED", + status: "DEPLOYED", + projectId: project.id, + environmentId: environment.id, + workerId: oldest.worker.id, + createdAt: new Date("2026-07-30T12:00:00.000Z"), + }, + }); + + expect(oldestDeployment.id > newestDeployment.id).toBe(true); + expect(oldestDeployment.createdAt < newestDeployment.createdAt).toBe(true); + + const resolver = new ControlPlaneResolver({ + controlPlaneReplica: prisma14, + controlPlanePrimary: prisma14, + cache: new ControlPlaneCache(), + splitEnabled: () => true, + }); + + const resolved = await resolver.resolveWorkerVersion({ + environmentId: environment.id, + type: "PRODUCTION", + }); + + expect(resolved).not.toBeNull(); + expect(resolved!.deployment!.id).toBe(newestDeployment.id); + expect(resolved!.worker.id).toBe(newest.worker.id); + } +); diff --git a/internal-packages/database/prisma/migrations/20260731160000_add_worker_deployment_environment_id_created_at_index/migration.sql b/internal-packages/database/prisma/migrations/20260731160000_add_worker_deployment_environment_id_created_at_index/migration.sql new file mode 100644 index 0000000000..57d9eec96c --- /dev/null +++ b/internal-packages/database/prisma/migrations/20260731160000_add_worker_deployment_environment_id_created_at_index/migration.sql @@ -0,0 +1 @@ +CREATE INDEX CONCURRENTLY IF NOT EXISTS "WorkerDeployment_environmentId_createdAt_idx" ON "public"."WorkerDeployment"("environmentId", "createdAt"); diff --git a/internal-packages/database/prisma/schema.prisma b/internal-packages/database/prisma/schema.prisma index 74c6d72a9c..ca1d868ab0 100644 --- a/internal-packages/database/prisma/schema.prisma +++ b/internal-packages/database/prisma/schema.prisma @@ -2208,6 +2208,7 @@ model WorkerDeployment { @@unique([projectId, shortCode]) @@unique([environmentId, version]) @@index([commitSHA]) + @@index([environmentId, createdAt]) } enum WorkerDeploymentStatus { diff --git a/internal-packages/run-engine/src/engine/controlPlaneResolver.ts b/internal-packages/run-engine/src/engine/controlPlaneResolver.ts index 89b0e15c48..da5f7b0060 100644 --- a/internal-packages/run-engine/src/engine/controlPlaneResolver.ts +++ b/internal-packages/run-engine/src/engine/controlPlaneResolver.ts @@ -234,9 +234,7 @@ export class PassthroughControlPlaneResolver implements ControlPlaneResolver { tasks: true, queues: true, }, - orderBy: { - id: "desc", - }, + orderBy: [{ createdAt: "desc" }, { id: "desc" }], }); if (!worker) { @@ -256,9 +254,6 @@ export class PassthroughControlPlaneResolver implements ControlPlaneResolver { tasks: true, queues: true, }, - orderBy: { - id: "desc", - }, }); if (!worker) { @@ -315,9 +310,7 @@ export class PassthroughControlPlaneResolver implements ControlPlaneResolver { environmentId, type: "MANAGED", }, - orderBy: { - id: "desc", - }, + orderBy: [{ createdAt: "desc" }, { id: "desc" }], include: { worker: { include: { diff --git a/internal-packages/run-engine/src/engine/tests/dequeueSystem.controlPlaneResolver.test.ts b/internal-packages/run-engine/src/engine/tests/dequeueSystem.controlPlaneResolver.test.ts index 5ede5c89db..3ea56bde8a 100644 --- a/internal-packages/run-engine/src/engine/tests/dequeueSystem.controlPlaneResolver.test.ts +++ b/internal-packages/run-engine/src/engine/tests/dequeueSystem.controlPlaneResolver.test.ts @@ -538,3 +538,142 @@ describe("DequeueSystem controlPlaneResolver (single-DB passthrough)", () => { } ); }); + +heteroPostgresTest( + "resolveWorkerVersion (DEVELOPMENT) resolves the newest worker by createdAt, not by id", + async ({ prisma14 }) => { + const cp = await seedControlPlane( + prisma14 as unknown as PrismaClient, + "cpord", + "ordering-task" + ); + + await prisma14.backgroundWorker.update({ + where: { id: cp.worker.id }, + data: { createdAt: new Date("2026-07-01T12:00:00.000Z") }, + }); + + const newest = await prisma14.backgroundWorker.create({ + data: { + friendlyId: generateFriendlyId("worker"), + contentHash: "hash_newest", + projectId: cp.project.id, + runtimeEnvironmentId: cp.environment.id, + version: "20260731.1", + metadata: {}, + engine: "V2", + createdAt: new Date("2026-07-31T12:00:00.000Z"), + }, + }); + const oldest = await prisma14.backgroundWorker.create({ + data: { + friendlyId: generateFriendlyId("worker"), + contentHash: "hash_oldest", + projectId: cp.project.id, + runtimeEnvironmentId: cp.environment.id, + version: "20260730.1", + metadata: {}, + engine: "V2", + createdAt: new Date("2026-07-30T12:00:00.000Z"), + }, + }); + + expect(oldest.id > newest.id).toBe(true); + expect(oldest.createdAt < newest.createdAt).toBe(true); + + const resolver = new PassthroughControlPlaneResolver({ + prisma: prisma14 as unknown as PrismaClient, + }); + + const resolved = await resolver.resolveWorkerVersion({ + environmentId: cp.environment.id, + type: "DEVELOPMENT", + }); + + assertNonNullable(resolved); + expect(resolved.worker.id).toBe(newest.id); + } +); + +heteroPostgresTest( + "resolveWorkerVersion latest-MANAGED fallback resolves by createdAt, not by id", + async ({ prisma14 }) => { + const cp = await seedControlPlane( + prisma14 as unknown as PrismaClient, + "cpfall", + "fallback-task" + ); + + await prisma14.workerDeployment.update({ + where: { id: cp.deployment.id }, + data: { type: "V1" }, + }); + + const newestWorker = await prisma14.backgroundWorker.create({ + data: { + friendlyId: generateFriendlyId("worker"), + contentHash: "hash_newest", + projectId: cp.project.id, + runtimeEnvironmentId: cp.environment.id, + version: "20260731.1", + metadata: {}, + engine: "V2", + }, + }); + const oldestWorker = await prisma14.backgroundWorker.create({ + data: { + friendlyId: generateFriendlyId("worker"), + contentHash: "hash_oldest", + projectId: cp.project.id, + runtimeEnvironmentId: cp.environment.id, + version: "20260730.1", + metadata: {}, + engine: "V2", + }, + }); + + const newestDeployment = await prisma14.workerDeployment.create({ + data: { + friendlyId: generateFriendlyId("deployment"), + contentHash: "hash_newest", + version: "20260731.1", + shortCode: "short_code_newest", + status: "DEPLOYED", + projectId: cp.project.id, + environmentId: cp.environment.id, + workerId: newestWorker.id, + type: "MANAGED", + createdAt: new Date("2026-07-31T12:00:00.000Z"), + }, + }); + const oldestDeployment = await prisma14.workerDeployment.create({ + data: { + friendlyId: generateFriendlyId("deployment"), + contentHash: "hash_oldest", + version: "20260730.1", + shortCode: "short_code_oldest", + status: "DEPLOYED", + projectId: cp.project.id, + environmentId: cp.environment.id, + workerId: oldestWorker.id, + type: "MANAGED", + createdAt: new Date("2026-07-30T12:00:00.000Z"), + }, + }); + + expect(oldestDeployment.id > newestDeployment.id).toBe(true); + expect(oldestDeployment.createdAt < newestDeployment.createdAt).toBe(true); + + const resolver = new PassthroughControlPlaneResolver({ + prisma: prisma14 as unknown as PrismaClient, + }); + + const resolved = await resolver.resolveWorkerVersion({ + environmentId: cp.environment.id, + type: "PRODUCTION", + }); + + assertNonNullable(resolved); + expect(resolved.deployment?.id).toBe(newestDeployment.id); + } +); From e93a16dd9b11bfe857f1ce7fb3da33a2af044a83 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Sat, 1 Aug 2026 09:15:36 +0100 Subject: [PATCH 2/2] fix(webapp): honor the caller's prisma client in the latest-deployment fallback findCurrentWorkerDeployment resolved the caller-supplied client for the promotion read but used the module-level client for the fallback read, so callers passing a replica silently read the primary for that one query. --- apps/webapp/app/v3/models/workerDeployment.server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/webapp/app/v3/models/workerDeployment.server.ts b/apps/webapp/app/v3/models/workerDeployment.server.ts index 62d7ea6313..48995bf2b2 100644 --- a/apps/webapp/app/v3/models/workerDeployment.server.ts +++ b/apps/webapp/app/v3/models/workerDeployment.server.ts @@ -128,7 +128,7 @@ export async function findCurrentWorkerDeployment({ } // We need to get the latest deployment of the given type - const latestDeployment = await prisma.workerDeployment.findFirst({ + const latestDeployment = await $prisma.workerDeployment.findFirst({ where: { environmentId, type,