diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 295289e9..59a77f49 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,6 +25,14 @@ concurrency: jobs: build-test: runs-on: ubuntu-latest + # the App server's Redis-backed stores (__tests__/adapters/app/redis.test.ts) run against a real server + services: + redis: + image: redis:7-alpine + ports: ['6379:6379'] + options: --health-cmd "redis-cli ping" --health-interval 5s --health-timeout 3s --health-retries 10 + env: + REDIS_TEST_URL: redis://127.0.0.1:6379 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 diff --git a/__tests__/adapters/app/lock.test.ts b/__tests__/adapters/app/lock.test.ts new file mode 100644 index 00000000..f027e7bd --- /dev/null +++ b/__tests__/adapters/app/lock.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from 'vitest' + +import { memoryLocks, pullRequestKey } from '../../../src/adapters/app/lock' + +function gate() { + let open!: () => void + const opened = new Promise((resolve) => { + open = resolve + }) + return { open, opened } +} + +describe('memoryLocks', () => { + it('runs two calls on one key one after the other, in order', async () => { + const locks = memoryLocks() + const order: string[] = [] + const first = gate() + + const a = locks.withLock('k', async () => { + order.push('a:start') + await first.opened + order.push('a:end') + return 'a' + }) + const b = locks.withLock('k', async () => { + order.push('b:start') + return 'b' + }) + await new Promise(resolve => setTimeout(resolve, 10)) + expect(order).toEqual(['a:start']) + + first.open() + expect(await Promise.all([a, b])).toEqual(['a', 'b']) + expect(order).toEqual(['a:start', 'a:end', 'b:start']) + }) + + it('runs different keys at the same time', async () => { + const locks = memoryLocks() + const order: string[] = [] + const first = gate() + + const a = locks.withLock('k1', async () => { + order.push('a:start') + await first.opened + }) + const b = locks.withLock('k2', async () => { + order.push('b:start') + }) + await b + expect(order).toEqual(['a:start', 'b:start']) + first.open() + await a + }) + + it('releases the lock when the holder throws, and the next holder still runs', async () => { + const locks = memoryLocks() + + const failed = locks.withLock('k', async () => { + throw new Error('boom') + }) + const next = locks.withLock('k', async () => 'ran') + + await expect(failed).rejects.toThrow('boom') + await expect(next).resolves.toBe('ran') + }) + + it('forgets a key once its last holder is done', async () => { + const locks = memoryLocks() + await locks.withLock('k', async () => {}) + // a fresh chain: this would hang if the previous one were still awaited incorrectly + await expect(locks.withLock('k', async () => 'again')).resolves.toBe('again') + }) +}) + +describe('pullRequestKey', () => { + const repository = { full_name: 'o/r', name: 'r', owner: { login: 'o' } } + + it('keys a pull request event by repository and number', () => { + expect(pullRequestKey({ id: 'd', eventName: 'pull_request', payload: { repository, pull_request: { number: 7 } } })).toBe('o/r#7') + }) + + it('keys an issue event the same way', () => { + expect(pullRequestKey({ id: 'd', eventName: 'issue_comment', payload: { repository, issue: { number: 3 } } })).toBe('o/r#3') + }) + + it('is undefined for an event about no issue, or without a repository', () => { + expect(pullRequestKey({ id: 'd', eventName: 'push', payload: { repository } })).toBeUndefined() + expect(pullRequestKey({ id: 'd', eventName: 'issues', payload: { issue: { number: 1 } } })).toBeUndefined() + }) +}) diff --git a/__tests__/adapters/app/redis.test.ts b/__tests__/adapters/app/redis.test.ts new file mode 100644 index 00000000..4d6ef6ac --- /dev/null +++ b/__tests__/adapters/app/redis.test.ts @@ -0,0 +1,139 @@ +import type { RedisLike } from '../../../src/adapters/app/redis' +import process from 'node:process' +import { createClient } from 'redis' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' + +import { redisDeliveryStore, redisLocks } from '../../../src/adapters/app/redis' +import * as sleepModule from '../../../src/utils/sleep' + +/** an in-memory SET NX / EVAL that is good enough for the two scripts the stores use */ +function fakeRedis(): RedisLike & { store: Map, calls: string[] } { + const store = new Map() + const calls: string[] = [] + return { + store, + calls, + async set(key, value, options) { + calls.push(`set ${key} ${JSON.stringify(options)}`) + if (options.NX && store.has(key)) { + return null + } + store.set(key, value) + return 'OK' + }, + async eval(_script, { keys, arguments: args }) { + calls.push(`eval ${keys[0]}`) + if (store.get(keys[0]!) === args[0]) { + store.delete(keys[0]!) + return 1 + } + return 0 + }, + } +} + +describe('redisDeliveryStore', () => { + it('reports an id the first time as new and every later time as seen, with the ttl', async () => { + const client = fakeRedis() + const store = redisDeliveryStore(client, 123) + + expect(await store.seen('a')).toBe(false) + expect(await store.seen('a')).toBe(true) + expect(await store.seen('b')).toBe(false) + expect(client.calls[0]).toBe('set prow:delivery:a {"NX":true,"EX":123}') + }) +}) + +describe('redisLocks', () => { + it('holds the key while fn runs and releases it by token afterwards', async () => { + const client = fakeRedis() + const locks = redisLocks(client, { ttlMs: 500 }) + + await locks.withLock('o/r#1', async () => { + expect(client.store.get('prow:lock:o/r#1')).toMatch(/^\d+-/) + }) + + expect(client.store.has('prow:lock:o/r#1')).toBe(false) + expect(client.calls.at(-1)).toBe('eval prow:lock:o/r#1') + }) + + it('waits for a held lock, then takes it', async () => { + const client = fakeRedis() + const sleep = vi.spyOn(sleepModule, 'sleep').mockImplementation(async () => { + client.store.delete('prow:lock:k') + }) + client.store.set('prow:lock:k', 'someone-else') + + await expect(redisLocks(client, { retryMs: 5 }).withLock('k', async () => 'got it')).resolves.toBe('got it') + expect(sleep).toHaveBeenCalledWith(5) + }) + + it('does not release a lock another holder took over after the ttl', async () => { + const client = fakeRedis() + const locks = redisLocks(client) + + await locks.withLock('k', async () => { + client.store.set('prow:lock:k', 'a-new-holder') + }) + + expect(client.store.get('prow:lock:k')).toBe('a-new-holder') + }) + + it('gives up after waitMs when the lock never frees, and still releases on a throwing fn', async () => { + const client = fakeRedis() + vi.spyOn(sleepModule, 'sleep').mockResolvedValue(undefined) + client.store.set('prow:lock:k', 'stuck') + + await expect(redisLocks(client, { waitMs: 1, retryMs: 1 }).withLock('k', async () => {})).rejects.toThrow(/could not lock k within 1ms/) + + client.store.clear() + await expect(redisLocks(client).withLock('k', async () => { + throw new Error('boom') + })).rejects.toThrow('boom') + expect(client.store.has('prow:lock:k')).toBe(false) + }) +}) + +// a real server: build-test provides one as a service container and sets REDIS_TEST_URL +describe.skipIf(!process.env.REDIS_TEST_URL)('against a redis server', () => { + const a = createClient({ url: process.env.REDIS_TEST_URL }) + const b = createClient({ url: process.env.REDIS_TEST_URL }) + const prefix = `t${Date.now()}` + + beforeAll(async () => { + await Promise.all([a.connect(), b.connect()]) + }) + afterAll(async () => { + await Promise.all([a.close(), b.close()]) + }) + + it('dedupes a delivery across two clients', async () => { + expect(await redisDeliveryStore(a).seen(`${prefix}-d`)).toBe(false) + expect(await redisDeliveryStore(b).seen(`${prefix}-d`)).toBe(true) + }) + + it('excludes two clients from one pull request at a time', async () => { + const order: string[] = [] + const la = redisLocks(a, { retryMs: 5 }) + const lb = redisLocks(b, { retryMs: 5 }) + await Promise.all([ + la.withLock(`${prefix}-pr`, async () => { + order.push('a:start') + await new Promise(resolve => setTimeout(resolve, 60)) + order.push('a:end') + }), + (async () => { + await new Promise(resolve => setTimeout(resolve, 10)) + await lb.withLock(`${prefix}-pr`, async () => { + order.push('b:start') + }) + })(), + ]) + expect(order).toEqual(['a:start', 'a:end', 'b:start']) + }) + + it('frees a lock whose holder vanished once the ttl passes', async () => { + await a.set(`prow:lock:${prefix}-dead`, 'gone', { NX: true, PX: 80 }) + await expect(redisLocks(b, { retryMs: 10, waitMs: 1000 }).withLock(`${prefix}-dead`, async () => 'recovered')).resolves.toBe('recovered') + }) +}) diff --git a/__tests__/server/stores.test.ts b/__tests__/server/stores.test.ts new file mode 100644 index 00000000..7f8418b7 --- /dev/null +++ b/__tests__/server/stores.test.ts @@ -0,0 +1,260 @@ +import { generateKeyPairSync } from 'node:crypto' +import { http } from 'msw' +import { setupServer } from 'msw/node' +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest' + +import { asyncLocalScope } from '../../src/adapters/app/scope' +import { receiveWebhook } from '../../src/adapters/app/webhook' +import { globalPlatformScope, installPlatformScope } from '../../src/platform' +import { buildApp, buildReceiver, main, memoryStores, redisStores, storesFromEnv } from '../../src/server/node' +import { signedRequest, withInstallation } from '../adapters/app/testApp' +import issueCommentEvent from '../fixtures/issues/issueCommentEvent.json' +import * as utils from '../testUtils' + +vi.mock('redis', () => ({ + createClient: vi.fn(() => { + const handlers = new Map void>() + return { + on: (event: string, handler: (e: unknown) => void) => handlers.set(event, handler), + connect: vi.fn(() => (globalThis as { __redisConnect?: Promise }).__redisConnect ?? Promise.resolve()), + close: vi.fn(async () => {}), + set: vi.fn(async () => 'OK'), + eval: vi.fn(async () => 1), + emit: (event: string, e: unknown) => handlers.get(event)?.(e), + } + }), +})) + +const api = 'https://api.github.com' +const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048, privateKeyEncoding: { type: 'pkcs1', format: 'pem' }, publicKeyEncoding: { type: 'pkcs1', format: 'pem' } }) +const secret = 'stores-secret' +const env = { GITHUB_APP_ID: '1', GITHUB_APP_PRIVATE_KEY: privateKey, GITHUB_WEBHOOK_SECRET: secret, PROW_COMMANDS: '/close' } + +const server = setupServer() +const unhandled = utils.failTestOnUnhandledRequests(server) + +describe('storesFromEnv', () => { + it('is memory with no REDIS_URL and says nothing for one replica', () => { + const lines: Record[] = [] + const stores = storesFromEnv({}, line => lines.push(line)) + expect(stores.deliveries).toBeDefined() + expect(lines).toEqual([]) + }) + + it('warns when more than one replica runs without a REDIS_URL', () => { + const lines: Record[] = [] + storesFromEnv({ PROW_REPLICAS: '3' }, line => lines.push(line)) + expect(lines).toEqual([expect.objectContaining({ level: 'warning', message: expect.stringContaining('PROW_REPLICAS=3 without REDIS_URL') })]) + }) + + it('connects to redis when REDIS_URL is set, logs the connection and redis errors, and closes', async () => { + const lines: Record[] = [] + const stores = storesFromEnv({ REDIS_URL: 'redis://x' }, line => lines.push(line)) + await stores.ready + expect(lines).toContainEqual({ level: 'info', message: 'redis: connected' }) + + const { createClient } = await import('redis') + const client = vi.mocked(createClient).mock.results.at(-1)!.value as { emit: (e: string, v: unknown) => void, close: () => Promise } + client.emit('error', new Error('ECONNRESET')) + client.emit('error', 'string error') + expect(lines).toContainEqual({ level: 'error', message: 'redis: ECONNRESET' }) + expect(lines).toContainEqual({ level: 'error', message: 'redis: string error' }) + + await stores.close() + expect(client.close).toHaveBeenCalled() + }) +}) + +describe('the server with stores', () => { + beforeAll(() => { + server.listen(utils.failOnUnhandledRequest) + installPlatformScope(asyncLocalScope()) + }) + afterEach(() => server.resetHandlers()) + afterAll(() => { + server.close() + installPlatformScope(globalPlatformScope) + }) + + it('/readyz is 503 until the stores are ready, then 200; /healthz is always 200', async () => { + let connect!: () => void + ;(globalThis as { __redisConnect?: Promise }).__redisConnect = new Promise((resolve) => { + connect = resolve + }) + try { + const stores = redisStores('redis://x', () => {}) + const app = buildApp(buildReceiver(env, stores, () => {}), stores) + + expect((await app.fetch(new Request('https://prow.test/healthz'))).status).toBe(200) + expect((await app.fetch(new Request('https://prow.test/readyz'))).status).toBe(503) + connect() + await stores.ready + expect((await app.fetch(new Request('https://prow.test/readyz'))).status).toBe(200) + } + finally { + delete (globalThis as { __redisConnect?: Promise }).__redisConnect + } + }) + + it('/readyz stays 503 when the stores never come up', async () => { + ;(globalThis as { __redisConnect?: Promise }).__redisConnect = Promise.reject(new Error('refused')) + try { + const stores = redisStores('redis://x', () => {}) + await stores.ready.catch(() => {}) + const app = buildApp(buildReceiver(env, stores, () => {}), stores) + await new Promise(resolve => setTimeout(resolve, 0)) + expect((await app.fetch(new Request('https://prow.test/readyz'))).status).toBe(503) + } + finally { + delete (globalThis as { __redisConnect?: Promise }).__redisConnect + } + }) + + it('runs two deliveries for one pull request one after the other, and tracks them in flight until done', async () => { + const stores = memoryStores() + const receiver = buildReceiver(env, stores, () => {}) + const app = buildApp(receiver, stores) + const order: string[] = [] + let releaseFirst!: () => void + const firstHeld = new Promise((resolve) => { + releaseFirst = resolve + }) + let patches = 0 + server.use( + http.post(`${api}/app/installations/7/access_tokens`, utils.mockResponse(201, { token: 't', expires_at: new Date(Date.now() + 3_600_000).toISOString(), permissions: {}, repository_selection: 'all' })), + http.patch(`${api}/repos/Codertocat/Hello-World/issues/1`, async () => { + patches++ + order.push(`patch:${patches}:start`) + if (patches === 1) { + await firstHeld + } + order.push(`patch:${patches}:end`) + return new Response('{}', { headers: { 'Content-Type': 'application/json' } }) + }), + ) + const payload = withInstallation(issueCommentEvent) + payload.comment.body = '/close' + + const r1 = await app.fetch(await signedRequest('issue_comment', payload, { secret, id: 'one' })) + const r2 = await app.fetch(await signedRequest('issue_comment', payload, { secret, id: 'two' })) + expect([r1.status, r2.status]).toEqual([202, 202]) + expect(receiver.inFlight.size).toBe(2) + await vi.waitFor(() => expect(order).toEqual(['patch:1:start'])) + await new Promise(resolve => setTimeout(resolve, 50)) + expect(order).toEqual(['patch:1:start']) + + releaseFirst() + await Promise.all([...receiver.inFlight]) + await new Promise(resolve => setTimeout(resolve, 0)) + + expect(order).toEqual(['patch:1:start', 'patch:1:end', 'patch:2:start', 'patch:2:end']) + expect(receiver.inFlight.size).toBe(0) + }) + + it('locks an event about no issue or pull request under its own delivery id', async () => { + const seen: string[] = [] + const stores = { ...memoryStores(), locks: { withLock: async (key: string, fn: () => Promise) => { + seen.push(key) + return fn() + } } } + const receiver = buildReceiver(env, stores, () => {}) + server.use(http.post(`${api}/app/installations/7/access_tokens`, utils.mockResponse(201, { token: 't', expires_at: new Date(Date.now() + 3_600_000).toISOString(), permissions: {}, repository_selection: 'all' }))) + // the push handler goes on to probe prow.yaml; the key is what this case is about + unhandled.allow() + + const { handled } = await receiveWebhook(receiver, await signedRequest('push', { ref: 'refs/heads/main', repository: issueCommentEvent.repository, installation: { id: 7 } }, { secret, id: 'push-1' })) + await handled + + expect(seen).toEqual(['delivery:push-1']) + }) + + it('main() drains in-flight deliveries on SIGTERM, closes the stores and exits 0', async () => { + const write = vi.spyOn(process.stdout, 'write').mockImplementation(() => true) + const signals = new (await import('node:events')).EventEmitter() as unknown as NodeJS.Process + const exit = vi.fn() + ;(signals as unknown as { exit: typeof exit }).exit = exit + const port = 20000 + Math.floor(Math.random() * 1000) + const { passthrough } = await import('msw') + server.use(http.all(`http://127.0.0.1:${port}/*`, () => passthrough())) + let releaseHandler!: () => void + const held = new Promise((resolve) => { + releaseHandler = resolve + }) + server.use( + http.post(`${api}/app/installations/7/access_tokens`, async () => { + await held + return Response.json({ token: 't', expires_at: new Date(Date.now() + 3_600_000).toISOString(), permissions: {}, repository_selection: 'all' }, { status: 201 }) + }), + http.get(`${api}/repos/Codertocat/Hello-World/collaborators/Codertocat`, utils.mockResponse(204)), + http.patch(`${api}/repos/Codertocat/Hello-World/issues/1`, utils.mockResponse(200, {})), + ) + const listening = main({ ...env, PORT: String(port), REDIS_URL: 'redis://x', PROW_SHUTDOWN_GRACE_MS: '5000' }, signals) + try { + await listening.ready + const payload = withInstallation(issueCommentEvent) + payload.comment.body = '/close' + const signed = await signedRequest('issue_comment', payload, { secret }) + const response = await fetch(`http://127.0.0.1:${port}/webhook`, { method: 'POST', headers: Object.fromEntries(signed.headers), body: await signed.text() }) + expect(response.status).toBe(202) + + signals.emit('SIGTERM') + await new Promise(resolve => setTimeout(resolve, 20)) + expect(exit).not.toHaveBeenCalled() + const logged = () => write.mock.calls.map(call => JSON.parse(String(call[0])) as { message: string }) + expect(logged().some(line => /waiting up to 5000ms for 1 in-flight/.test(line.message))).toBe(true) + + releaseHandler() + await vi.waitFor(() => expect(exit).toHaveBeenCalledWith(0)) + expect(logged().at(-1)).toMatchObject({ message: 'shut down' }) + } + finally { + await listening.close(0) + write.mockRestore() + } + }) + + it('main() gives up waiting after the grace period', async () => { + const write = vi.spyOn(process.stdout, 'write').mockImplementation(() => true) + const signals = new (await import('node:events')).EventEmitter() as unknown as NodeJS.Process + const exit = vi.fn() + ;(signals as unknown as { exit: typeof exit }).exit = exit + const port = 21000 + Math.floor(Math.random() * 1000) + const { passthrough } = await import('msw') + server.use(http.all(`http://127.0.0.1:${port}/*`, () => passthrough())) + server.use(http.post(`${api}/app/installations/7/access_tokens`, () => new Promise(() => {}))) + const listening = main({ ...env, PORT: String(port), PROW_SHUTDOWN_GRACE_MS: '30' }, signals) + try { + await listening.ready + const payload = withInstallation(issueCommentEvent) + payload.comment.body = '/close' + const signed = await signedRequest('issue_comment', payload, { secret }) + await fetch(`http://127.0.0.1:${port}/webhook`, { method: 'POST', headers: Object.fromEntries(signed.headers), body: await signed.text() }) + + signals.emit('SIGINT') + await vi.waitFor(() => expect(exit).toHaveBeenCalledWith(0)) + } + finally { + write.mockRestore() + } + }) + + it('main() exits 1 when closing the stores fails', async () => { + const write = vi.spyOn(process.stdout, 'write').mockImplementation(() => true) + const signals = new (await import('node:events')).EventEmitter() as unknown as NodeJS.Process + const exit = vi.fn() + ;(signals as unknown as { exit: typeof exit }).exit = exit + const port = 22000 + Math.floor(Math.random() * 1000) + const listening = main({ ...env, PORT: String(port), REDIS_URL: 'redis://x' }, signals) + try { + await listening.ready + const { createClient } = await import('redis') + const client = vi.mocked(createClient).mock.results.at(-1)!.value as { close: ReturnType } + client.close.mockRejectedValueOnce(new Error('cannot close')) + signals.emit('SIGTERM') + await vi.waitFor(() => expect(exit).toHaveBeenCalledWith(1)) + } + finally { + write.mockRestore() + } + }) +}) diff --git a/docs/github-app.md b/docs/github-app.md index 4f8d57bb..7baa6745 100644 --- a/docs/github-app.md +++ b/docs/github-app.md @@ -47,6 +47,8 @@ the organization (all repositories, or the ones Prow should manage). | `GITHUB_WEBHOOK_SECRET` | yes | the webhook secret | | `GITHUB_API_URL` | no | the REST root for GitHub Enterprise Server, e.g. `https://ghes.example/api/v3` | | `PORT` | no | listening port, default `8080` | +| `REDIS_URL` | for several replicas | shared dedupe and locks, see [One replica or several](#one-replica-or-several) | +| `PROW_SHUTDOWN_GRACE_MS` | no | how long `SIGTERM` waits for in-flight events, default `25000` | | `PROW_COMMANDS` | no | same as the action's `prow-commands` input | | `PROW_JOBS` | no | same as `jobs`; what the scheduler runs. Default `lgtm` | | `PROW_CRON_INTERVAL` | no | minutes between scheduled runs inside the server; unset means none (use the `cron` command) | @@ -70,14 +72,31 @@ docker run -p 8080:8080 \ prow-app ``` -`GET /healthz` answers `ok`. `POST /webhook` verifies `X-Hub-Signature-256`, -answers `202` at once and handles the event afterwards; a redelivered -`X-GitHub-Delivery` id is handled once per process. Run one replica: the -delivery memory and the per-event ordering are in-process. For local testing, -relay deliveries with [smee.io](https://smee.io) or `gh webhook forward`. +`GET /healthz` answers `ok` (liveness); `GET /readyz` answers `ok` once the +stores below are reachable (readiness). `POST /webhook` verifies +`X-Hub-Signature-256`, answers `202` at once and handles the event afterwards. +A pull request's events run one at a time, in order, as the action's +`concurrency:` group arranges; a redelivered `X-GitHub-Delivery` id is handled +once. On `SIGTERM` the server stops accepting, lets in-flight events finish +(`PROW_SHUTDOWN_GRACE_MS`, default 25 000) and exits. For local testing, relay +deliveries with [smee.io](https://smee.io) or `gh webhook forward`. + +### One replica or several + +| | one replica | several replicas | +|---|---|---| +| delivery dedupe and the per-pull-request lock | in process | in a Redis every replica shares | +| configuration | nothing | `REDIS_URL=redis://host:6379` (or `rediss://`) | + +Without `REDIS_URL`, two replicas would each handle a redelivery and could race +on one pull request; the server warns at startup when `PROW_REPLICAS` is above +1 and `REDIS_URL` is unset. Any Redis 6+ or Valkey works; the server uses two +keys per event (`prow:delivery:`, a day; `prow:lock:`, a +minute at most) and nothing else. A minimal Kubernetes Deployment is one container with the variables above from a Secret and a Service in front of port 8080; put your usual Ingress/TLS in front. +A Helm chart is on its way. ## Scheduled jobs diff --git a/eslint.config.mjs b/eslint.config.mjs index e080dcde..cb6a59c8 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -8,6 +8,7 @@ export default antfu({ '**/lib/**/', 'node_modules/', '**/node_modules/**/', + '.sisyphus/', ], }, { files: ['__tests__/**/*.test.ts'], diff --git a/package-lock.json b/package-lock.json index 1c2fca68..c4e7199b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,8 @@ "@octokit/rest": "^22.0.1", "@octokit/webhooks-methods": "^6.0.0", "hono": "^4.13.14", - "js-yaml": "^5.5.0" + "js-yaml": "^5.5.0", + "redis": "^6.3.1" }, "devDependencies": { "@antfu/eslint-config": "^9.5.1", @@ -3736,6 +3737,75 @@ "dev": true, "license": "MIT" }, + "node_modules/@redis/bloom": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-6.3.1.tgz", + "integrity": "sha512-dpF9j2yJDoPzSCaZn0MSB3xzmnXQVJeTSAWJcQVcgbmluwhpFeemmII0cF1t+WYfoVS+shEFq79+MmUItgzJLg==", + "license": "MIT", + "engines": { + "node": ">= 20.0.0" + }, + "peerDependencies": { + "@redis/client": "^6.3.1" + } + }, + "node_modules/@redis/client": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-6.3.1.tgz", + "integrity": "sha512-masbfVKULdoAofwddepZxRhWa4w6zvBHn6TRn90TRPrRkVIlGCur3pFqwWaCbMxtYI9twepkz63e/T/tEH8GjA==", + "license": "MIT", + "engines": { + "node": ">= 20.0.0" + }, + "peerDependencies": { + "@node-rs/xxhash": "^1.1.0", + "@opentelemetry/api": ">=1 <2" + }, + "peerDependenciesMeta": { + "@node-rs/xxhash": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + } + } + }, + "node_modules/@redis/json": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-6.3.1.tgz", + "integrity": "sha512-fMz4kUwJzEftcU9EMaqsjZcUwzpTQLOYRqhQsAD6uKvZWeamblK13ZhgO5SzVSZVk31jmwLeTRpU5GAl5egfUw==", + "license": "MIT", + "engines": { + "node": ">= 20.0.0" + }, + "peerDependencies": { + "@redis/client": "^6.3.1" + } + }, + "node_modules/@redis/search": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-6.3.1.tgz", + "integrity": "sha512-T67pvPUpW82VrZLvihO8O2DFMC7eY2oMAzMzapCbiPZn7mdWqHMEIlohYim12gIawT5kn6qT/Cvmm4WulKB+kQ==", + "license": "MIT", + "engines": { + "node": ">= 20.0.0" + }, + "peerDependencies": { + "@redis/client": "^6.3.1" + } + }, + "node_modules/@redis/time-series": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-6.3.1.tgz", + "integrity": "sha512-MNDgrm+i63M/1a+XA+HcDsScwt4uBh/fGGJ84swTQZTFK5K5wMmxdlTRa4ZgmNJzjxMaszW/ipPScMUGV3aAMg==", + "license": "MIT", + "engines": { + "node": ">= 20.0.0" + }, + "peerDependencies": { + "@redis/client": "^6.3.1" + } + }, "node_modules/@rolldown/binding-android-arm-eabi": { "version": "1.2.12", "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.12.tgz", @@ -9436,6 +9506,22 @@ "url": "https://github.com/sindresorhus/quote-js-string?sponsor=1" } }, + "node_modules/redis": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/redis/-/redis-6.3.1.tgz", + "integrity": "sha512-mV96ZKxQ4CGn3SyEGaY0Lb9YA443mdVey3c+mdr0NAUaBxiOZGoYCWX8PoM55kictSOmxlJRtZJxIsmZm3jL+w==", + "license": "MIT", + "dependencies": { + "@redis/bloom": "6.3.1", + "@redis/client": "6.3.1", + "@redis/json": "6.3.1", + "@redis/search": "6.3.1", + "@redis/time-series": "6.3.1" + }, + "engines": { + "node": ">= 20.0.0" + } + }, "node_modules/refa": { "version": "0.12.1", "resolved": "https://registry.npmjs.org/refa/-/refa-0.12.1.tgz", diff --git a/package.json b/package.json index 92b11bf8..048b7fe8 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,8 @@ "@octokit/rest": "^22.0.1", "@octokit/webhooks-methods": "^6.0.0", "hono": "^4.13.14", - "js-yaml": "^5.5.0" + "js-yaml": "^5.5.0", + "redis": "^6.3.1" }, "devDependencies": { "@antfu/eslint-config": "^9.5.1", diff --git a/src/adapters/app/lock.ts b/src/adapters/app/lock.ts new file mode 100644 index 00000000..25328dd1 --- /dev/null +++ b/src/adapters/app/lock.ts @@ -0,0 +1,41 @@ +import type { Delivery } from './platform' + +/** + * PullRequestLocks runs the events of one pull request one after another, as + * the action's `concurrency:` group does, so that two events arriving together + * do not race the merge gate. + */ +export interface PullRequestLocks { + withLock: (key: string, fn: () => Promise) => Promise +} + +/** the lock key of a delivery: `owner/repo#number`, or undefined for an event about no issue or pull request */ +export function pullRequestKey(delivery: Delivery): string | undefined { + const number: unknown = delivery.payload.pull_request?.number ?? delivery.payload.issue?.number + const repository = delivery.payload.repository + if (typeof number !== 'number' || !repository?.full_name) { + return undefined + } + return `${repository.full_name}#${number}` +} + +/** one process: a promise chain per key */ +export function memoryLocks(): PullRequestLocks { + const tails = new Map>() + return { + async withLock(key, fn) { + const previous = tails.get(key) ?? Promise.resolve() + const own = previous.then(fn, fn) + const settled = own.then(() => undefined, () => undefined) + tails.set(key, settled) + try { + return await own + } + finally { + if (tails.get(key) === settled) { + tails.delete(key) + } + } + }, + } +} diff --git a/src/adapters/app/redis.ts b/src/adapters/app/redis.ts new file mode 100644 index 00000000..1b5082e0 --- /dev/null +++ b/src/adapters/app/redis.ts @@ -0,0 +1,58 @@ +import type { DeliveryStore } from './dedupe' +import type { PullRequestLocks } from './lock' + +import { sleep } from '../../utils/sleep' + +/** the subset of a node-redis client these stores use, so tests can fake it */ +export interface RedisLike { + set: (key: string, value: string, options: { NX: true, PX?: number, EX?: number }) => Promise + eval: (script: string, options: { keys: string[], arguments: string[] }) => Promise +} + +/** delivery ids shared by every replica; `SET NX` makes the first writer the only handler */ +export function redisDeliveryStore(client: RedisLike, ttlSeconds = 86_400): DeliveryStore { + return { + async seen(id) { + const reply = await client.set(`prow:delivery:${id}`, '1', { NX: true, EX: ttlSeconds }) + return reply === null + }, + } +} + +export interface RedisLockOptions { + /** how long a holder may keep the lock before it is presumed dead; default 60 s */ + ttlMs?: number + /** how long to wait between acquisition attempts; default 50 ms */ + retryMs?: number + /** give up after this long; default the ttl, so a dead holder's lock is waited out exactly once */ + waitMs?: number +} + +// release only what we hold: compare the token, then delete, in one step on the server +const releaseScript = 'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end' + +/** a lock per pull request shared by every replica, taken with `SET NX PX` and released by its holder */ +export function redisLocks(client: RedisLike, options: RedisLockOptions = {}): PullRequestLocks { + const ttlMs = options.ttlMs ?? 60_000 + const retryMs = options.retryMs ?? 50 + const waitMs = options.waitMs ?? ttlMs + return { + async withLock(key, fn) { + const name = `prow:lock:${key}` + const token = `${Date.now()}-${Math.random().toString(36).slice(2)}` + const deadline = Date.now() + waitMs + while (await client.set(name, token, { NX: true, PX: ttlMs }) === null) { + if (Date.now() >= deadline) { + throw new Error(`could not lock ${key} within ${waitMs}ms`) + } + await sleep(retryMs) + } + try { + return await fn() + } + finally { + await client.eval(releaseScript, { keys: [name], arguments: [token] }) + } + }, + } +} diff --git a/src/server/node.ts b/src/server/node.ts index 8e40a910..0cd13bfc 100644 --- a/src/server/node.ts +++ b/src/server/node.ts @@ -1,13 +1,18 @@ import type { Scheduler } from '../adapters/app/cron' import type { DeliveryStore } from '../adapters/app/dedupe' +import type { PullRequestLocks } from '../adapters/app/lock' +import type { LogSink } from '../adapters/app/platform' import type { WebhookReceiver } from '../adapters/app/webhook' import process from 'node:process' import { serve } from '@hono/node-server' import { Hono } from 'hono' +import { createClient } from 'redis' import { appInstallationLister, runScheduledJobs } from '../adapters/app/cron' import { memoryDeliveryStore } from '../adapters/app/dedupe' +import { memoryLocks, pullRequestKey } from '../adapters/app/lock' import { appTokenSource } from '../adapters/app/platform' +import { redisDeliveryStore, redisLocks } from '../adapters/app/redis' import { asyncLocalScope } from '../adapters/app/scope' import { settingsFromEnv } from '../adapters/app/settings' import { receiveWebhook } from '../adapters/app/webhook' @@ -15,16 +20,72 @@ import { handleCronJobs } from '../cronJobs/handleCronJob' import { installPlatformScope } from '../platform' import { run } from '../run' -export function buildReceiver(env: Record, deliveries: DeliveryStore = memoryDeliveryStore()): WebhookReceiver { +/** what the replicas of a deployment share: nothing (one replica) or a Redis */ +export interface Stores { + deliveries: DeliveryStore + locks: PullRequestLocks + /** resolves once the backing service answers; readiness waits on it */ + ready: Promise + close: () => Promise +} + +export function memoryStores(): Stores { + return { deliveries: memoryDeliveryStore(), locks: memoryLocks(), ready: Promise.resolve(), close: async () => {} } +} + +export function redisStores(url: string, sink: LogSink): Stores { + const client = createClient({ url }) + client.on('error', (error: unknown) => sink({ level: 'error', message: `redis: ${error instanceof Error ? error.message : String(error)}` })) + const ready = client.connect().then(() => { + sink({ level: 'info', message: 'redis: connected' }) + }) + return { + deliveries: redisDeliveryStore(client), + locks: redisLocks(client), + ready, + close: async () => { + await client.close() + }, + } +} + +/** + * storesFromEnv picks the shared stores: REDIS_URL set means every replica + * shares deliveries and locks through it; unset means this process is on its + * own, which is only right for a single replica. + */ +export function storesFromEnv(env: Record, sink: LogSink): Stores { + if (env.REDIS_URL) { + return redisStores(env.REDIS_URL, sink) + } + if (Number(env.PROW_REPLICAS ?? 1) > 1) { + sink({ level: 'warning', message: `PROW_REPLICAS=${env.PROW_REPLICAS} without REDIS_URL: replicas will handle redeliveries twice and race on a pull request; set REDIS_URL` }) + } + return memoryStores() +} + +export function jsonStdoutSink(): LogSink { + return line => process.stdout.write(`${JSON.stringify({ time: new Date().toISOString(), ...line })}\n`) +} + +export interface Receiver extends WebhookReceiver { + /** every accepted delivery still being handled; drained on shutdown */ + inFlight: Set> +} + +export function buildReceiver(env: Record, stores: Stores = memoryStores(), sink: LogSink = jsonStdoutSink()): Receiver { const settings = settingsFromEnv(env) + const inFlight = new Set>() return { settings, tokens: appTokenSource(settings), - deliveries, - sink: line => process.stdout.write(`${JSON.stringify({ time: new Date().toISOString(), ...line })}\n`), - dispatch: run, + deliveries: stores.deliveries, + sink, + inFlight, + dispatch: (context, delivery) => stores.locks.withLock(pullRequestKey(delivery) ?? `delivery:${delivery.id}`, () => run(context)), background: (work) => { - void work + inFlight.add(work) + void work.finally(() => inFlight.delete(work)) }, } } @@ -46,9 +107,14 @@ export async function runCronOnce(env: Record = proc return report.failed.length === 0 ? 0 : 1 } -export function buildApp(receiver: WebhookReceiver): Hono { +export function buildApp(receiver: WebhookReceiver, stores: Stores = memoryStores()): Hono { const app = new Hono() + let storesReady = false + stores.ready.then(() => { + storesReady = true + }, () => {}) app.get('/healthz', c => c.text('ok')) + app.get('/readyz', c => (storesReady ? c.text('ok') : c.text('stores not ready', 503))) app.post('/webhook', async (c) => { const { response } = await receiveWebhook(receiver, c.req.raw) return response @@ -58,19 +124,22 @@ export function buildApp(receiver: WebhookReceiver): Hono { export interface Listening { ready: Promise - close: () => void + /** stop listening, let in-flight deliveries finish (bounded by graceMs), close the stores */ + close: (graceMs?: number) => Promise } -export function main(env: Record = process.env): Listening { +export function main(env: Record = process.env, signals: NodeJS.Process = process): Listening { installPlatformScope(asyncLocalScope()) - const receiver = buildReceiver(env) + const sink = jsonStdoutSink() + const stores = storesFromEnv(env, sink) + const receiver = buildReceiver(env, stores, sink) const port = Number(env.PORT ?? 8080) let resolveReady!: () => void const ready = new Promise((resolve) => { resolveReady = resolve }) - const server = serve({ fetch: buildApp(receiver).fetch, port }, () => { - receiver.sink({ level: 'info', message: `listening on :${port}` }) + const server = serve({ fetch: buildApp(receiver, stores).fetch, port }, () => { + sink({ level: 'info', message: `listening on :${port}` }) resolveReady() }) // PROW_CRON_INTERVAL: run the scheduled jobs in-process every N minutes, for a single-container deployment @@ -79,15 +148,31 @@ export function main(env: Record = process.env): Lis const timer = minutes > 0 ? setInterval(() => { runScheduledJobs(scheduler).catch((error) => { - receiver.sink({ level: 'error', event: 'schedule', message: error instanceof Error ? error.message : String(error) }) + sink({ level: 'error', event: 'schedule', message: error instanceof Error ? error.message : String(error) }) }) }, minutes * 60_000) : undefined - return { - ready, - close: () => { + + let closing: Promise | undefined + const close = (graceMs = Number(env.PROW_SHUTDOWN_GRACE_MS ?? 25_000)): Promise => { + closing ??= (async () => { clearInterval(timer) server.close() - }, + const pending = receiver.inFlight.size + if (pending > 0) { + sink({ level: 'info', message: `shutting down: waiting up to ${graceMs}ms for ${pending} in-flight deliveries` }) + await Promise.race([Promise.allSettled([...receiver.inFlight]), new Promise(resolve => setTimeout(resolve, graceMs))]) + } + await stores.close() + sink({ level: 'info', message: 'shut down' }) + })() + return closing + } + const onSignal = (): void => { + close().then(() => signals.exit(0), () => signals.exit(1)) } + signals.once('SIGTERM', onSignal) + signals.once('SIGINT', onSignal) + + return { ready, close } } diff --git a/src/server/worker.ts b/src/server/worker.ts index b6374529..3f44a912 100644 --- a/src/server/worker.ts +++ b/src/server/worker.ts @@ -6,6 +6,7 @@ import { DurableObject } from 'cloudflare:workers' import { Hono } from 'hono' import { appInstallationLister, runScheduledJobs } from '../adapters/app/cron' +import { pullRequestKey } from '../adapters/app/lock' import { appPlatform, appTokenSource } from '../adapters/app/platform' import { asyncLocalScope } from '../adapters/app/scope' import { settingsFromEnv } from '../adapters/app/settings' @@ -61,15 +62,6 @@ export class PullRequestLock extends DurableObject { } } -export function pullRequestKey(delivery: Delivery): string | undefined { - const number: unknown = delivery.payload.pull_request?.number ?? delivery.payload.issue?.number - const repository = delivery.payload.repository - if (typeof number !== 'number' || !repository?.full_name) { - return undefined - } - return `${repository.full_name}#${number}` -} - // Workers Logs capture every console method; `info` keeps the lines at their own level, apart from runtime errors // eslint-disable-next-line no-console const emit = (line: Record): void => console.info(JSON.stringify(line))