From 8630d649100a5e103243661480bf18a58cb03995 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Wed, 19 Aug 2026 11:02:53 +0200 Subject: [PATCH 1/3] refactor(storage): extract webhook matcher config --- lambdas/functions/webhook/package.json | 1 + .../webhook/src/ConfigLoader.test.ts | 163 +++++------------- lambdas/functions/webhook/src/ConfigLoader.ts | 42 +---- lambdas/functions/webhook/src/lambda.test.ts | 15 +- lambdas/functions/webhook/src/modules.d.ts | 1 - .../webhook/src/runners/dispatch.test.ts | 26 ++- .../webhook/src/webhook/index.test.ts | 22 ++- .../aws/ssm/environment.d.ts | 1 + .../ssm/runner-matcher-config-store.test.ts | 103 +++++++++++ .../aws/ssm/runner-matcher-config-store.ts | 69 ++++++++ lambdas/libs/storage-providers/core/index.ts | 4 + lambdas/libs/storage-providers/index.ts | 2 + .../runner-matcher-config.test.ts | 83 +++++++++ .../runner-matcher-config.ts | 23 +++ .../libs/storage-providers/vitest.config.ts | 1 + lambdas/yarn.lock | 1 + 16 files changed, 369 insertions(+), 188 deletions(-) create mode 100644 lambdas/libs/storage-providers/aws/ssm/runner-matcher-config-store.test.ts create mode 100644 lambdas/libs/storage-providers/aws/ssm/runner-matcher-config-store.ts create mode 100644 lambdas/libs/storage-providers/runner-matcher-config.test.ts create mode 100644 lambdas/libs/storage-providers/runner-matcher-config.ts diff --git a/lambdas/functions/webhook/package.json b/lambdas/functions/webhook/package.json index 34f4ef3de9..83d9810d1f 100644 --- a/lambdas/functions/webhook/package.json +++ b/lambdas/functions/webhook/package.json @@ -31,6 +31,7 @@ "@aws-github-runner/aws-powertools-util": "*", "@aws-github-runner/aws-ssm-util": "*", "@aws-github-runner/compute-providers": "*", + "@aws-github-runner/storage-providers": "*", "@aws-sdk/client-sqs": "^3.1009.0", "@middy/core": "^6.4.5", "@octokit/rest": "22.0.1", diff --git a/lambdas/functions/webhook/src/ConfigLoader.test.ts b/lambdas/functions/webhook/src/ConfigLoader.test.ts index 9f4e5e5864..7c7aa1dcf7 100644 --- a/lambdas/functions/webhook/src/ConfigLoader.test.ts +++ b/lambdas/functions/webhook/src/ConfigLoader.test.ts @@ -1,4 +1,5 @@ -import { getParameter, getParameters } from '@aws-github-runner/aws-ssm-util'; +import { getParameter } from '@aws-github-runner/aws-ssm-util'; +import { getRunnerMatcherConfigStore, type RunnerMatcherConfigStore } from '@aws-github-runner/storage-providers'; import { ConfigWebhook, ConfigWebhookEventBridge, ConfigDispatcher } from './ConfigLoader'; import { logger } from '@aws-github-runner/aws-powertools-util'; @@ -6,6 +7,11 @@ import { RunnerMatcherConfig } from './sqs'; import { describe, it, expect, beforeEach, vi } from 'vitest'; vi.mock('@aws-github-runner/aws-ssm-util'); +vi.mock('@aws-github-runner/storage-providers'); + +const runnerMatcherConfigStore = { + get: vi.fn(), +} satisfies RunnerMatcherConfigStore; describe('ConfigLoader Tests', () => { beforeEach(() => { @@ -14,6 +20,7 @@ describe('ConfigLoader Tests', () => { ConfigWebhookEventBridge.reset(); ConfigDispatcher.reset(); logger.setLogLevel('DEBUG'); + vi.mocked(getRunnerMatcherConfigStore).mockReturnValue(runnerMatcherConfigStore); // clear process.env for (const key of Object.keys(process.env)) { @@ -24,7 +31,6 @@ describe('ConfigLoader Tests', () => { describe('Check base object', () => { function setupConfiguration(): void { process.env.EVENT_BUS_NAME = 'event-bus'; - process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config'; process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; const matcherConfig = [ { @@ -36,15 +42,8 @@ describe('ConfigLoader Tests', () => { }, }, ]; - vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { - if (paramPath === '/path/to/matcher/config') { - return JSON.stringify(matcherConfig); - } - if (paramPath === '/path/to/webhook/secret') { - return 'secret'; - } - return ''; - }); + runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(matcherConfig)); + vi.mocked(getParameter).mockResolvedValue('secret'); } it('should return the same instance of ConfigWebhook (singleton)', async () => { @@ -53,7 +52,8 @@ describe('ConfigLoader Tests', () => { const config2 = await ConfigWebhook.load(); expect(config1).toBe(config2); - expect(getParameter).toHaveBeenCalledTimes(2); + expect(getParameter).toHaveBeenCalledOnce(); + expect(runnerMatcherConfigStore.get).toHaveBeenCalledOnce(); }); it('should return the same instance of ConfigWebhookEventBridge (singleton)', async () => { @@ -63,6 +63,7 @@ describe('ConfigLoader Tests', () => { expect(config1).toBe(config2); expect(getParameter).toHaveBeenCalledTimes(1); + expect(runnerMatcherConfigStore.get).not.toHaveBeenCalled(); }); it('should return the same instance of ConfigDispatcher (singleton)', async () => { @@ -71,7 +72,8 @@ describe('ConfigLoader Tests', () => { const config2 = await ConfigDispatcher.load(); expect(config1).toBe(config2); - expect(getParameter).toHaveBeenCalledTimes(1); + expect(getParameter).not.toHaveBeenCalled(); + expect(runnerMatcherConfigStore.get).toHaveBeenCalledOnce(); }); it('should filter secrets from being logged', async () => { @@ -95,7 +97,6 @@ describe('ConfigLoader Tests', () => { it('should load config successfully', async () => { process.env.REPOSITORY_ALLOW_LIST = '["repo1", "repo2"]'; process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; - process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config'; const matcherConfig = [ { id: '1', @@ -106,15 +107,8 @@ describe('ConfigLoader Tests', () => { }, }, ]; - vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { - if (paramPath === '/path/to/matcher/config') { - return JSON.stringify(matcherConfig); - } - if (paramPath === '/path/to/webhook/secret') { - return 'secret'; - } - return ''; - }); + runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(matcherConfig)); + vi.mocked(getParameter).mockResolvedValue('secret'); const config: ConfigWebhook = await ConfigWebhook.load(); @@ -124,7 +118,6 @@ describe('ConfigLoader Tests', () => { }); it('should load config successfully', async () => { - process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config'; process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; const matcherConfig = [ { @@ -136,15 +129,8 @@ describe('ConfigLoader Tests', () => { }, }, ]; - vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { - if (paramPath === '/path/to/matcher/config') { - return JSON.stringify(matcherConfig); - } - if (paramPath === '/path/to/webhook/secret') { - return 'secret'; - } - return ''; - }); + runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(matcherConfig)); + vi.mocked(getParameter).mockResolvedValue('secret'); const config: ConfigWebhook = await ConfigWebhook.load(); @@ -155,46 +141,27 @@ describe('ConfigLoader Tests', () => { }); it('should throw error if config loading fails', async () => { - process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config'; - - vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { - if (paramPath === '/path/to/matcher/config') { - throw new Error('Failed to load matcher config'); - } - return ''; - }); + runnerMatcherConfigStore.get.mockRejectedValue( + new Error( + 'Failed to load parameter for matcherConfig from path /path/to/matcher/config: Failed to load matcher config', + ), + ); + vi.mocked(getParameter).mockResolvedValue(''); await expect(ConfigWebhook.load()).rejects.toThrow( 'Failed to load config: Failed to load parameter for matcherConfig from path /path/to/matcher/config: Failed to load matcher config', ); }); - it('should load config successfully from multiple paths', async () => { - process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config-1:/path/to/matcher/config-2'; + it('should load combined matcher config returned by the store', async () => { process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; - const partialMatcher1 = - '[{"id":"1","arn":"arn:aws:sqs:queue1","matcherConfig":{"labelMatchers":[["a"]],"exactMatch":true}}'; - const partialMatcher2 = - ',{"id":"2","arn":"arn:aws:sqs:queue2","matcherConfig":{"labelMatchers":[["b"]],"exactMatch":true}}]'; - const combinedMatcherConfig = [ { id: '1', arn: 'arn:aws:sqs:queue1', matcherConfig: { labelMatchers: [['a']], exactMatch: true } }, { id: '2', arn: 'arn:aws:sqs:queue2', matcherConfig: { labelMatchers: [['b']], exactMatch: true } }, ]; - - // Mock getParameters for batch fetching multiple paths - vi.mocked(getParameters).mockResolvedValue( - new Map([ - ['/path/to/matcher/config-1', partialMatcher1], - ['/path/to/matcher/config-2', partialMatcher2], - ]), - ); - - vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { - if (paramPath === '/path/to/webhook/secret') return 'secret'; - return ''; - }); + runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(combinedMatcherConfig)); + vi.mocked(getParameter).mockResolvedValue('secret'); const config: ConfigWebhook = await ConfigWebhook.load(); @@ -202,27 +169,14 @@ describe('ConfigLoader Tests', () => { expect(config.webhookSecret).toBe('secret'); }); - it('should throw error if config loading fails from multiple paths', async () => { - process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config-1:/path/to/matcher/config-2'; + it('should propagate an error from the matcher config store', async () => { process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; - - const partialMatcher1 = - '[{"id":"1","arn":"arn:aws:sqs:queue1","matcherConfig":{"labelMatchers":[["a"]],"exactMatch":true}}'; - const partialMatcher2 = - ',{"id":"2","arn":"arn:aws:sqs:queue2","matcherConfig":{"labelMatchers":[["b"]],"exactMatch":true}}'; - - // Mock getParameters for batch fetching - returns incomplete JSON that will fail to parse - vi.mocked(getParameters).mockResolvedValue( - new Map([ - ['/path/to/matcher/config-1', partialMatcher1], - ['/path/to/matcher/config-2', partialMatcher2], - ]), + runnerMatcherConfigStore.get.mockRejectedValue( + new Error( + "Failed to load/parse combined matcher config: Expected ',' or ']' after array element in JSON at position 196", + ), ); - - vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { - if (paramPath === '/path/to/webhook/secret') return 'secret'; - return ''; - }); + vi.mocked(getParameter).mockResolvedValue('secret'); await expect(ConfigWebhook.load()).rejects.toThrow( "Failed to load config: Failed to load/parse combined matcher config: Expected ',' or ']' after array element in JSON at position 196", @@ -248,6 +202,7 @@ describe('ConfigLoader Tests', () => { expect(config.allowedEvents).toEqual(['push', 'pull_request']); expect(config.eventBusName).toBe('event-bus'); expect(config.webhookSecret).toBe('secret'); + expect(runnerMatcherConfigStore.get).not.toHaveBeenCalled(); }); it('should throw error if config loading fails', async () => { @@ -264,7 +219,6 @@ describe('ConfigLoader Tests', () => { describe('ConfigDispatcher', () => { it('should load config successfully', async () => { process.env.REPOSITORY_ALLOW_LIST = '["repo1", "repo2"]'; - process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config'; const matcherConfig: RunnerMatcherConfig[] = [ { @@ -276,12 +230,7 @@ describe('ConfigLoader Tests', () => { }, }, ]; - vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { - if (paramPath === '/path/to/matcher/config') { - return JSON.stringify(matcherConfig); - } - return ''; - }); + runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(matcherConfig)); const config: ConfigDispatcher = await ConfigDispatcher.load(); @@ -289,27 +238,14 @@ describe('ConfigLoader Tests', () => { expect(config.matcherConfig).toEqual(matcherConfig); }); - it('should load config successfully from multiple paths with repo allow list', async () => { + it('should load combined matcher config returned by the store with repo allow list', async () => { process.env.REPOSITORY_ALLOW_LIST = '["repo1", "repo2"]'; - process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config-1:/path/to/matcher/config-2'; - - const partial1 = - '[{"id":"1","arn":"arn:aws:sqs:queue1","matcherConfig":{"labelMatchers":[["x"]],"exactMatch":true}}'; - const partial2 = - ',{"id":"2","arn":"arn:aws:sqs:queue2","matcherConfig":{"labelMatchers":[["y"]],"exactMatch":true}}]'; const combined: RunnerMatcherConfig[] = [ { id: '1', arn: 'arn:aws:sqs:queue1', matcherConfig: { labelMatchers: [['x']], exactMatch: true } }, { id: '2', arn: 'arn:aws:sqs:queue2', matcherConfig: { labelMatchers: [['y']], exactMatch: true } }, ]; - - // Mock getParameters for batch fetching multiple paths - vi.mocked(getParameters).mockResolvedValue( - new Map([ - ['/path/to/matcher/config-1', partial1], - ['/path/to/matcher/config-2', partial2], - ]), - ); + runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(combined)); const config: ConfigDispatcher = await ConfigDispatcher.load(); @@ -318,18 +254,15 @@ describe('ConfigLoader Tests', () => { }); it('should throw error if config loading fails', async () => { - vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { - throw new Error(`Parameter ${paramPath} not found`); - }); + runnerMatcherConfigStore.get.mockRejectedValue(new Error('Matcher config store is unavailable')); await expect(ConfigDispatcher.load()).rejects.toThrow( - 'Failed to load config: Failed to load parameter for matcherConfig from path undefined: Parameter undefined not found', + 'Failed to load config: Matcher config store is unavailable', ); }); it('should rely on default when optionals are not set.', async () => { process.env.ACCEPT_EVENTS = 'null'; - process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config'; const matcherConfig: RunnerMatcherConfig[] = [ { arn: 'arn:aws:sqs:eu-central-1:123456:npalm-default-queued-builds', @@ -340,12 +273,7 @@ describe('ConfigLoader Tests', () => { }, }, ]; - vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { - if (paramPath === '/path/to/matcher/config') { - return JSON.stringify(matcherConfig); - } - return ''; - }); + runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(matcherConfig)); const config: ConfigDispatcher = await ConfigDispatcher.load(); @@ -355,14 +283,7 @@ describe('ConfigLoader Tests', () => { it('should throw an error if runner matcher config is empty.', async () => { process.env.REPOSITORY_ALLOW_LIST = '["repo1", "repo2"]'; - process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config'; - - vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { - if (paramPath === '/path/to/matcher/config') { - return JSON.stringify(''); - } - return ''; - }); + runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify('')); await expect(ConfigDispatcher.load()).rejects.toThrow('Failed to load config: Matcher config is empty'); }); diff --git a/lambdas/functions/webhook/src/ConfigLoader.ts b/lambdas/functions/webhook/src/ConfigLoader.ts index d9d9da2590..2cf261b849 100644 --- a/lambdas/functions/webhook/src/ConfigLoader.ts +++ b/lambdas/functions/webhook/src/ConfigLoader.ts @@ -1,4 +1,5 @@ -import { getParameter, getParameters } from '@aws-github-runner/aws-ssm-util'; +import { getParameter } from '@aws-github-runner/aws-ssm-util'; +import { getRunnerMatcherConfigStore } from '@aws-github-runner/storage-providers'; import { RunnerMatcherConfig } from './sqs'; import { logger } from '@aws-github-runner/aws-powertools-util'; @@ -66,7 +67,7 @@ abstract class BaseConfig { }); } - private loadProperty(propertyName: keyof this, value: string) { + protected loadProperty(propertyName: keyof this, value: string) { try { this[propertyName] = JSON.parse(value) as unknown as this[keyof this]; } catch { @@ -96,38 +97,11 @@ abstract class MatcherAwareConfig extends BaseConfig { // across the matching queues to avoid concentrating load on a single one. queueSelectionStrategy: QueueSelectionStrategy = 'first'; - protected async loadMatcherConfig(paramPathsEnv: string) { - if (!paramPathsEnv || paramPathsEnv === 'undefined' || paramPathsEnv === 'null' || !paramPathsEnv.includes(':')) { - // Single path or invalid string → load directly - await this.loadParameter(paramPathsEnv, 'matcherConfig'); - return; - } - - const paths = paramPathsEnv - .split(':') - .map((p) => p.trim()) - .filter(Boolean); - - // Batch fetch all matcher config paths in a single SSM API call + protected async loadMatcherConfig() { try { - const params = await getParameters(paths); - let combinedString = ''; - for (const path of paths) { - const value = params.get(path); - if (value) { - combinedString += value; - } else { - this.configLoadingErrors.push( - `Failed to load parameter for matcherConfig from path ${path}: Parameter not found`, - ); - } - } - - if (combinedString) { - this.matcherConfig = JSON.parse(combinedString); - } + this.loadProperty('matcherConfig', await getRunnerMatcherConfigStore().get()); } catch (error) { - this.configLoadingErrors.push(`Failed to load/parse combined matcher config: ${(error as Error).message}`); + this.configLoadingErrors.push((error as Error).message); } } } @@ -142,7 +116,7 @@ export class ConfigWebhook extends MatcherAwareConfig { this.loadEnvVar(process.env.QUEUE_SELECTION_STRATEGY, 'queueSelectionStrategy', 'first'); await Promise.all([ - this.loadMatcherConfig(process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH), + this.loadMatcherConfig(), this.loadParameter(process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET, 'webhookSecret'), ]); @@ -174,7 +148,7 @@ export class ConfigDispatcher extends MatcherAwareConfig { async loadConfig(): Promise { this.loadEnvVar(process.env.REPOSITORY_ALLOW_LIST, 'repositoryAllowList', []); this.loadEnvVar(process.env.QUEUE_SELECTION_STRATEGY, 'queueSelectionStrategy', 'first'); - await this.loadMatcherConfig(process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH); + await this.loadMatcherConfig(); validateRunnerMatcherConfig(this); validateQueueSelectionStrategy(this); diff --git a/lambdas/functions/webhook/src/lambda.test.ts b/lambdas/functions/webhook/src/lambda.test.ts index d65b8371c4..3bc67e42dd 100644 --- a/lambdas/functions/webhook/src/lambda.test.ts +++ b/lambdas/functions/webhook/src/lambda.test.ts @@ -7,6 +7,7 @@ import { dispatchToRunners, eventBridgeWebhook, directWebhook } from './lambda'; import { publishForRunners, publishOnEventBridge } from './webhook'; import ValidationError from './ValidationError'; import { getParameter } from '@aws-github-runner/aws-ssm-util'; +import { getRunnerMatcherConfigStore, type RunnerMatcherConfigStore } from '@aws-github-runner/storage-providers'; import { dispatch } from './runners/dispatch'; import { EventWrapper } from './types'; import { describe, it, expect, beforeEach, vi } from 'vitest'; @@ -80,14 +81,20 @@ const context: Context = { vi.mock('./runners/dispatch'); vi.mock('./webhook'); vi.mock('@aws-github-runner/aws-ssm-util'); +vi.mock('@aws-github-runner/storage-providers'); + +const runnerMatcherConfigStore = { + get: vi.fn(), +} satisfies RunnerMatcherConfigStore; describe('Test webhook lambda wrapper.', () => { beforeEach(() => { - // We mock all SSM request to resolve to a non empty array. Since we mock all implemeantions - // relying on the config object that is enough to test the handlers. - const mockedGet = vi.mocked(getParameter); - mockedGet.mockResolvedValue('["abc"]'); vi.clearAllMocks(); + // The handlers only need non-empty config values because their downstream + // implementations are mocked in this wrapper test. + vi.mocked(getParameter).mockResolvedValue('["abc"]'); + vi.mocked(getRunnerMatcherConfigStore).mockReturnValue(runnerMatcherConfigStore); + runnerMatcherConfigStore.get.mockResolvedValue('["abc"]'); }); describe('Test webhook lambda wrapper.', () => { diff --git a/lambdas/functions/webhook/src/modules.d.ts b/lambdas/functions/webhook/src/modules.d.ts index 9110746709..3b04a2a5be 100644 --- a/lambdas/functions/webhook/src/modules.d.ts +++ b/lambdas/functions/webhook/src/modules.d.ts @@ -3,7 +3,6 @@ declare namespace NodeJS { ENVIRONMENT: string; EVENT_BUS_NAME: string; PARAMETER_GITHUB_APP_WEBHOOK_SECRET: string; - PARAMETER_RUNNER_MATCHER_CONFIG_PATH: string; QUEUE_SELECTION_STRATEGY: string; REPOSITORY_ALLOW_LIST: string; RUNNER_LABELS: string; diff --git a/lambdas/functions/webhook/src/runners/dispatch.test.ts b/lambdas/functions/webhook/src/runners/dispatch.test.ts index bb2cdc7cce..b3140d6e96 100644 --- a/lambdas/functions/webhook/src/runners/dispatch.test.ts +++ b/lambdas/functions/webhook/src/runners/dispatch.test.ts @@ -1,5 +1,5 @@ -import { getParameter } from '@aws-github-runner/aws-ssm-util'; import { selectDynamicLabelQueue } from '@aws-github-runner/compute-providers/webhook'; +import { getRunnerMatcherConfigStore, type RunnerMatcherConfigStore } from '@aws-github-runner/storage-providers'; import nock from 'nock'; import { WorkflowJobEvent } from '@octokit/webhooks-types'; @@ -14,12 +14,14 @@ import { logger } from '@aws-github-runner/aws-powertools-util'; import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; vi.mock('../sqs'); -vi.mock('@aws-github-runner/aws-ssm-util'); +vi.mock('@aws-github-runner/storage-providers'); vi.mock('@aws-github-runner/compute-providers/webhook', () => ({ selectDynamicLabelQueue: vi.fn(), })); -const GITHUB_APP_WEBHOOK_SECRET = 'TEST_SECRET'; +const runnerMatcherConfigStore = { + get: vi.fn(), +} satisfies RunnerMatcherConfigStore; const cleanEnv = process.env; @@ -37,7 +39,6 @@ describe('Dispatcher', () => { vi.clearAllMocks(); vi.resetAllMocks(); - mockSSMResponse(); config = await createConfig(undefined, runnerConfig); }); @@ -242,7 +243,7 @@ describe('Dispatcher', () => { it('rejects an invalid strategy at config load', async () => { process.env.QUEUE_SELECTION_STRATEGY = 'bogus'; ConfigDispatcher.reset(); - mockSSMResponse(twoExactMatches); + mockMatcherConfigResponse(twoExactMatches); await expect(ConfigDispatcher.load()).rejects.toThrow(/queue selection strategy/i); }); }); @@ -394,16 +395,9 @@ describe('Dispatcher', () => { }); }); -function mockSSMResponse(runnerConfigInput?: RunnerConfig) { - process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/github-runner/runner-matcher-config'; - const mockedGet = vi.mocked(getParameter); - mockedGet.mockImplementation((parameter_name) => { - const value = - parameter_name == '/github-runner/runner-matcher-config' - ? JSON.stringify(runnerConfigInput ?? runnerConfig) - : GITHUB_APP_WEBHOOK_SECRET; - return Promise.resolve(value); - }); +function mockMatcherConfigResponse(runnerConfigInput?: RunnerConfig) { + vi.mocked(getRunnerMatcherConfigStore).mockReturnValue(runnerMatcherConfigStore); + runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(runnerConfigInput ?? runnerConfig)); } async function createConfig(repositoryAllowList?: string[], runnerConfig?: RunnerConfig): Promise { @@ -411,6 +405,6 @@ async function createConfig(repositoryAllowList?: string[], runnerConfig?: Runne process.env.REPOSITORY_ALLOW_LIST = JSON.stringify(repositoryAllowList); } ConfigDispatcher.reset(); - mockSSMResponse(runnerConfig); + mockMatcherConfigResponse(runnerConfig); return await ConfigDispatcher.load(); } diff --git a/lambdas/functions/webhook/src/webhook/index.test.ts b/lambdas/functions/webhook/src/webhook/index.test.ts index aa4fbbc506..43345388f7 100644 --- a/lambdas/functions/webhook/src/webhook/index.test.ts +++ b/lambdas/functions/webhook/src/webhook/index.test.ts @@ -1,5 +1,6 @@ import { Webhooks } from '@octokit/webhooks'; import { getParameter } from '@aws-github-runner/aws-ssm-util'; +import { getRunnerMatcherConfigStore, type RunnerMatcherConfigStore } from '@aws-github-runner/storage-providers'; import nock from 'nock'; @@ -16,8 +17,12 @@ vi.mock('../sqs'); vi.mock('../eventbridge'); vi.mock('../runners/dispatch'); vi.mock('@aws-github-runner/aws-ssm-util'); +vi.mock('@aws-github-runner/storage-providers'); const GITHUB_APP_WEBHOOK_SECRET = 'TEST_SECRET'; +const runnerMatcherConfigStore = { + get: vi.fn(), +} satisfies RunnerMatcherConfigStore; const cleanEnv = process.env; @@ -32,7 +37,7 @@ describe('handle GitHub webhook events', () => { nock.disableNetConnect(); vi.clearAllMocks(); - mockSSMResponse(); + mockConfigResponse(); }); describe('handle and dispatch webhook events to build queues', () => { @@ -284,8 +289,7 @@ describe('Check message size (checkBodySize)', () => { }); }); -function mockSSMResponse() { - process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config'; +function mockConfigResponse() { process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; const matcherConfig = [ { @@ -297,13 +301,7 @@ function mockSSMResponse() { }, }, ]; - vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { - if (paramPath === '/path/to/matcher/config') { - return JSON.stringify(matcherConfig); - } - if (paramPath === '/path/to/webhook/secret') { - return GITHUB_APP_WEBHOOK_SECRET; - } - throw new Error('Parameter not found'); - }); + vi.mocked(getRunnerMatcherConfigStore).mockReturnValue(runnerMatcherConfigStore); + runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(matcherConfig)); + vi.mocked(getParameter).mockResolvedValue(GITHUB_APP_WEBHOOK_SECRET); } diff --git a/lambdas/libs/storage-providers/aws/ssm/environment.d.ts b/lambdas/libs/storage-providers/aws/ssm/environment.d.ts index 2604ce057e..5ffa7b85bb 100644 --- a/lambdas/libs/storage-providers/aws/ssm/environment.d.ts +++ b/lambdas/libs/storage-providers/aws/ssm/environment.d.ts @@ -3,6 +3,7 @@ export {}; declare global { namespace NodeJS { interface ProcessEnv { + PARAMETER_RUNNER_MATCHER_CONFIG_PATH?: string; SSM_PARAMETER_STORE_TAGS?: string; SSM_CONFIG_PATH?: string; SSM_TOKEN_PATH?: string; diff --git a/lambdas/libs/storage-providers/aws/ssm/runner-matcher-config-store.test.ts b/lambdas/libs/storage-providers/aws/ssm/runner-matcher-config-store.test.ts new file mode 100644 index 0000000000..4b9d2f3379 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm/runner-matcher-config-store.test.ts @@ -0,0 +1,103 @@ +import { getParameter, getParameters } from '@aws-github-runner/aws-ssm-util'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAwsSsmRunnerMatcherConfigStore } from './runner-matcher-config-store'; + +vi.mock('@aws-github-runner/aws-ssm-util', () => ({ + getParameter: vi.fn(), + getParameters: vi.fn(), +})); + +const getParameterMock = vi.mocked(getParameter); +const getParametersMock = vi.mocked(getParameters); +const cleanEnv = process.env; + +describe('aws_ssm runner matcher config store', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env = { ...cleanEnv }; + delete process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH; + }); + + it('loads a single matcher config parameter', async () => { + process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/runner/matcher/config'; + getParameterMock.mockResolvedValue('[{"id":"runner"}]'); + + await expect(createAwsSsmRunnerMatcherConfigStore().get()).resolves.toBe('[{"id":"runner"}]'); + + expect(getParameterMock).toHaveBeenCalledWith('/runner/matcher/config'); + expect(getParametersMock).not.toHaveBeenCalled(); + }); + + it('loads and concatenates matcher config chunks in configured order', async () => { + process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = ' /runner/matcher/1 : : /runner/matcher/2 '; + getParametersMock.mockResolvedValue( + new Map([ + ['/runner/matcher/2', ',{"id":"runner-2"}]'], + ['/runner/matcher/1', '[{"id":"runner-1"}'], + ]), + ); + + await expect(createAwsSsmRunnerMatcherConfigStore().get()).resolves.toBe('[{"id":"runner-1"},{"id":"runner-2"}]'); + + expect(getParametersMock).toHaveBeenCalledWith(['/runner/matcher/1', '/runner/matcher/2']); + expect(getParameterMock).not.toHaveBeenCalled(); + }); + + it('rejects a missing matcher config chunk', async () => { + process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/runner/matcher/1:/runner/matcher/2'; + getParametersMock.mockResolvedValue(new Map([['/runner/matcher/1', '[{"id":"runner-1"}']])); + + await expect(createAwsSsmRunnerMatcherConfigStore().get()).rejects.toThrow( + 'Failed to load parameter for matcherConfig from path /runner/matcher/2: Parameter not found', + ); + }); + + it('rejects malformed combined matcher config', async () => { + process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/runner/matcher/1:/runner/matcher/2'; + getParametersMock.mockResolvedValue( + new Map([ + ['/runner/matcher/1', '[{"id":"runner-1"}'], + ['/runner/matcher/2', ',{"id":"runner-2"}'], + ]), + ); + + await expect(createAwsSsmRunnerMatcherConfigStore().get()).rejects.toThrow( + "Failed to load/parse combined matcher config: Expected ',' or ']' after array element", + ); + }); + + it('propagates a single parameter read failure', async () => { + process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/runner/matcher/config'; + const error = new Error('read failed'); + getParameterMock.mockRejectedValue(error); + + await expect(createAwsSsmRunnerMatcherConfigStore().get()).rejects.toThrow( + 'Failed to load parameter for matcherConfig from path /runner/matcher/config: read failed', + ); + }); + + it('propagates a batch parameter read failure', async () => { + process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/runner/matcher/1:/runner/matcher/2'; + const error = new Error('read failed'); + getParametersMock.mockRejectedValue(error); + + await expect(createAwsSsmRunnerMatcherConfigStore().get()).rejects.toThrow( + 'Failed to load/parse combined matcher config: read failed', + ); + }); + + it.each([undefined, '', ' '])('requires matcher config parameter paths for input %j', (parameterPaths) => { + if (parameterPaths === undefined) { + delete process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH; + } else { + process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = parameterPaths; + } + + expect(() => createAwsSsmRunnerMatcherConfigStore()).toThrow( + 'Environment variable PARAMETER_RUNNER_MATCHER_CONFIG_PATH is not set', + ); + expect(getParameterMock).not.toHaveBeenCalled(); + expect(getParametersMock).not.toHaveBeenCalled(); + }); +}); diff --git a/lambdas/libs/storage-providers/aws/ssm/runner-matcher-config-store.ts b/lambdas/libs/storage-providers/aws/ssm/runner-matcher-config-store.ts new file mode 100644 index 0000000000..166151b850 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm/runner-matcher-config-store.ts @@ -0,0 +1,69 @@ +import { getParameter, getParameters } from '@aws-github-runner/aws-ssm-util'; + +import type { RunnerMatcherConfigStore } from '../../core'; +import type {} from './environment'; + +export function createAwsSsmRunnerMatcherConfigStore(): RunnerMatcherConfigStore { + const parameterPaths = process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH; + if (!parameterPaths || parameterPaths.trim() === '') { + throw new Error('Environment variable PARAMETER_RUNNER_MATCHER_CONFIG_PATH is not set'); + } + + const paths = parameterPaths + .split(':') + .map((path) => path.trim()) + .filter(Boolean); + + if (paths.length === 0) { + throw new Error('Environment variable PARAMETER_RUNNER_MATCHER_CONFIG_PATH is not set'); + } + + return new AwsSsmRunnerMatcherConfigStore(paths); +} + +class AwsSsmRunnerMatcherConfigStore implements RunnerMatcherConfigStore { + constructor(private readonly parameterPaths: string[]) {} + + async get(): Promise { + if (this.parameterPaths.length === 1) { + const path = this.parameterPaths[0]; + try { + return await getParameter(path); + } catch (error) { + throw new Error(`Failed to load parameter for matcherConfig from path ${path}: ${(error as Error).message}`); + } + } + + let parameters: Map; + try { + parameters = await getParameters(this.parameterPaths); + } catch (error) { + throw new Error(`Failed to load/parse combined matcher config: ${(error as Error).message}`); + } + + let combined = ''; + const errors: string[] = []; + for (const path of this.parameterPaths) { + const value = parameters.get(path); + if (value) { + combined += value; + } else { + errors.push(`Failed to load parameter for matcherConfig from path ${path}: Parameter not found`); + } + } + + if (combined) { + try { + JSON.parse(combined); + } catch (error) { + errors.push(`Failed to load/parse combined matcher config: ${(error as Error).message}`); + } + } + + if (errors.length > 0) { + throw new Error(errors.join(', ')); + } + + return combined; + } +} diff --git a/lambdas/libs/storage-providers/core/index.ts b/lambdas/libs/storage-providers/core/index.ts index 27339a2159..3d9fc43fe8 100644 --- a/lambdas/libs/storage-providers/core/index.ts +++ b/lambdas/libs/storage-providers/core/index.ts @@ -58,3 +58,7 @@ export interface RunnerGroupCacheStore { get(runnerGroupName: string): Promise; create(record: RunnerGroupCacheRecord): Promise; } + +export interface RunnerMatcherConfigStore { + get(): Promise; +} diff --git a/lambdas/libs/storage-providers/index.ts b/lambdas/libs/storage-providers/index.ts index 4b5f14f1c6..026b79d48a 100644 --- a/lambdas/libs/storage-providers/index.ts +++ b/lambdas/libs/storage-providers/index.ts @@ -9,6 +9,7 @@ export type { RunnerConfigStore, RunnerGroupCacheRecord, RunnerGroupCacheStore, + RunnerMatcherConfigStore, } from './core'; export { createRunnerConfigHousekeeper } from './runner-config-housekeeper'; export { createRunnerConfigConsumer, type RunnerConfigConsumerConfig } from './runner-config-consumer'; @@ -20,3 +21,4 @@ export { export type { RunnerConfigStorageProvider } from './provider'; export { createCommonStorage, createStorageProviders } from './storage-providers'; export type { StorageProviders, RunnerConfigStorage, CommonStorage } from './core'; +export { getRunnerMatcherConfigStore, resetRunnerMatcherConfigStore } from './runner-matcher-config'; diff --git a/lambdas/libs/storage-providers/runner-matcher-config.test.ts b/lambdas/libs/storage-providers/runner-matcher-config.test.ts new file mode 100644 index 0000000000..0dd7f42ad3 --- /dev/null +++ b/lambdas/libs/storage-providers/runner-matcher-config.test.ts @@ -0,0 +1,83 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAwsSsmRunnerMatcherConfigStore } from './aws/ssm/runner-matcher-config-store'; +import type { RunnerMatcherConfigStore } from './core'; +import { getRunnerMatcherConfigStore, resetRunnerMatcherConfigStore } from './runner-matcher-config'; + +vi.mock('./aws/ssm/runner-matcher-config-store', () => ({ + createAwsSsmRunnerMatcherConfigStore: vi.fn(), +})); + +const createAwsSsmRunnerMatcherConfigStoreMock = vi.mocked(createAwsSsmRunnerMatcherConfigStore); +const cleanEnv = process.env; + +describe('runner matcher config store selection', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env = { ...cleanEnv }; + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + resetRunnerMatcherConfigStore(); + }); + + it.each([undefined, '', ' '])('uses aws_ssm for default selector input %j', (provider) => { + setProvider(provider); + const store = stubStore(); + + expect(getRunnerMatcherConfigStore()).toBe(store); + expect(createAwsSsmRunnerMatcherConfigStoreMock).toHaveBeenCalledOnce(); + }); + + it.each(['aws_ssm', ' AWS_SSM '])('uses aws_ssm for explicit selector input %j', (provider) => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + const store = stubStore(); + + expect(getRunnerMatcherConfigStore()).toBe(store); + expect(createAwsSsmRunnerMatcherConfigStoreMock).toHaveBeenCalledOnce(); + }); + + it('rejects an unsupported provider on first use', () => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = 'not-registered'; + + expect(() => getRunnerMatcherConfigStore()).toThrow("Unsupported runner config storage provider 'not-registered'"); + expect(createAwsSsmRunnerMatcherConfigStoreMock).not.toHaveBeenCalled(); + }); + + it('selects lazily and caches the created store', () => { + const store = stubStore(); + + expect(createAwsSsmRunnerMatcherConfigStoreMock).not.toHaveBeenCalled(); + const first = getRunnerMatcherConfigStore(); + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = 'not-registered'; + const second = getRunnerMatcherConfigStore(); + + expect(first).toBe(store); + expect(second).toBe(store); + expect(createAwsSsmRunnerMatcherConfigStoreMock).toHaveBeenCalledOnce(); + }); + + it('selects again after the test reset', () => { + const firstStore = stubStore(); + expect(getRunnerMatcherConfigStore()).toBe(firstStore); + + const secondStore = { get: vi.fn() } satisfies RunnerMatcherConfigStore; + createAwsSsmRunnerMatcherConfigStoreMock.mockReturnValue(secondStore); + resetRunnerMatcherConfigStore(); + + expect(getRunnerMatcherConfigStore()).toBe(secondStore); + expect(createAwsSsmRunnerMatcherConfigStoreMock).toHaveBeenCalledTimes(2); + }); +}); + +function setProvider(provider: string | undefined): void { + if (provider === undefined) { + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + } else { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + } +} + +function stubStore(): RunnerMatcherConfigStore { + const store = { get: vi.fn() } satisfies RunnerMatcherConfigStore; + createAwsSsmRunnerMatcherConfigStoreMock.mockReturnValue(store); + return store; +} diff --git a/lambdas/libs/storage-providers/runner-matcher-config.ts b/lambdas/libs/storage-providers/runner-matcher-config.ts new file mode 100644 index 0000000000..6d56d49754 --- /dev/null +++ b/lambdas/libs/storage-providers/runner-matcher-config.ts @@ -0,0 +1,23 @@ +import { createAwsSsmRunnerMatcherConfigStore } from './aws/ssm/runner-matcher-config-store'; +import type { RunnerMatcherConfigStore } from './core'; +import type {} from './environment'; +import { resolveRunnerConfigStorageProvider, type RunnerConfigStorageProvider } from './provider'; + +type RunnerMatcherConfigStoreFactory = () => RunnerMatcherConfigStore; + +const providerFactories = { + aws_ssm: createAwsSsmRunnerMatcherConfigStore, +} as const satisfies Record; + +let runnerMatcherConfigStore: RunnerMatcherConfigStore | undefined; + +export function getRunnerMatcherConfigStore(): RunnerMatcherConfigStore { + runnerMatcherConfigStore ??= + providerFactories[resolveRunnerConfigStorageProvider(process.env.RUNNER_CONFIG_STORAGE_PROVIDER)](); + return runnerMatcherConfigStore; +} + +// Test-only reset for cases that need to exercise first-use environment selection. +export function resetRunnerMatcherConfigStore(): void { + runnerMatcherConfigStore = undefined; +} diff --git a/lambdas/libs/storage-providers/vitest.config.ts b/lambdas/libs/storage-providers/vitest.config.ts index 20c739f253..f107f07411 100644 --- a/lambdas/libs/storage-providers/vitest.config.ts +++ b/lambdas/libs/storage-providers/vitest.config.ts @@ -13,6 +13,7 @@ export default mergeConfig(defaultConfig, { 'runner-config-consumer.ts', 'storage-providers.ts', 'provider.ts', + 'runner-matcher-config.ts', 'core/**/*.ts', 'aws/**/*.ts', ], diff --git a/lambdas/yarn.lock b/lambdas/yarn.lock index 0e7daca0c7..49dbdd1d66 100644 --- a/lambdas/yarn.lock +++ b/lambdas/yarn.lock @@ -243,6 +243,7 @@ __metadata: "@aws-github-runner/aws-powertools-util": "npm:*" "@aws-github-runner/aws-ssm-util": "npm:*" "@aws-github-runner/compute-providers": "npm:*" + "@aws-github-runner/storage-providers": "npm:*" "@aws-sdk/client-eventbridge": "npm:^3.1009.0" "@aws-sdk/client-sqs": "npm:^3.1009.0" "@middy/core": "npm:^6.4.5" From d12860ad0f9a22dc3241bef758c97b5b8c87bb87 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Wed, 19 Aug 2026 11:21:36 +0200 Subject: [PATCH 2/3] refactor(storage): extract webhook secret store --- lambdas/functions/webhook/package.json | 1 - .../webhook/src/ConfigLoader.test.ts | 62 +++++++------- lambdas/functions/webhook/src/ConfigLoader.ts | 25 +++--- lambdas/functions/webhook/src/lambda.test.ts | 15 +++- lambdas/functions/webhook/src/modules.d.ts | 1 - .../webhook/src/webhook/index.test.ts | 16 ++-- .../aws/ssm/environment.d.ts | 1 + .../ssm/github-webhook-secret-store.test.ts | 51 ++++++++++++ .../aws/ssm/github-webhook-secret-store.ts | 27 ++++++ lambdas/libs/storage-providers/core/index.ts | 3 + .../github-webhook-secret.test.ts | 83 +++++++++++++++++++ .../github-webhook-secret.ts | 23 +++++ lambdas/libs/storage-providers/index.ts | 2 + .../libs/storage-providers/vitest.config.ts | 1 + lambdas/yarn.lock | 1 - 15 files changed, 256 insertions(+), 56 deletions(-) create mode 100644 lambdas/libs/storage-providers/aws/ssm/github-webhook-secret-store.test.ts create mode 100644 lambdas/libs/storage-providers/aws/ssm/github-webhook-secret-store.ts create mode 100644 lambdas/libs/storage-providers/github-webhook-secret.test.ts create mode 100644 lambdas/libs/storage-providers/github-webhook-secret.ts diff --git a/lambdas/functions/webhook/package.json b/lambdas/functions/webhook/package.json index 83d9810d1f..c596db3493 100644 --- a/lambdas/functions/webhook/package.json +++ b/lambdas/functions/webhook/package.json @@ -29,7 +29,6 @@ }, "dependencies": { "@aws-github-runner/aws-powertools-util": "*", - "@aws-github-runner/aws-ssm-util": "*", "@aws-github-runner/compute-providers": "*", "@aws-github-runner/storage-providers": "*", "@aws-sdk/client-sqs": "^3.1009.0", diff --git a/lambdas/functions/webhook/src/ConfigLoader.test.ts b/lambdas/functions/webhook/src/ConfigLoader.test.ts index 7c7aa1dcf7..41bc66f13b 100644 --- a/lambdas/functions/webhook/src/ConfigLoader.test.ts +++ b/lambdas/functions/webhook/src/ConfigLoader.test.ts @@ -1,14 +1,20 @@ -import { getParameter } from '@aws-github-runner/aws-ssm-util'; -import { getRunnerMatcherConfigStore, type RunnerMatcherConfigStore } from '@aws-github-runner/storage-providers'; +import { + getGitHubWebhookSecretStore, + getRunnerMatcherConfigStore, + type GitHubWebhookSecretStore, + type RunnerMatcherConfigStore, +} from '@aws-github-runner/storage-providers'; import { ConfigWebhook, ConfigWebhookEventBridge, ConfigDispatcher } from './ConfigLoader'; import { logger } from '@aws-github-runner/aws-powertools-util'; import { RunnerMatcherConfig } from './sqs'; import { describe, it, expect, beforeEach, vi } from 'vitest'; -vi.mock('@aws-github-runner/aws-ssm-util'); vi.mock('@aws-github-runner/storage-providers'); +const githubWebhookSecretStore = { + get: vi.fn(), +} satisfies GitHubWebhookSecretStore; const runnerMatcherConfigStore = { get: vi.fn(), } satisfies RunnerMatcherConfigStore; @@ -20,6 +26,7 @@ describe('ConfigLoader Tests', () => { ConfigWebhookEventBridge.reset(); ConfigDispatcher.reset(); logger.setLogLevel('DEBUG'); + vi.mocked(getGitHubWebhookSecretStore).mockReturnValue(githubWebhookSecretStore); vi.mocked(getRunnerMatcherConfigStore).mockReturnValue(runnerMatcherConfigStore); // clear process.env @@ -31,7 +38,6 @@ describe('ConfigLoader Tests', () => { describe('Check base object', () => { function setupConfiguration(): void { process.env.EVENT_BUS_NAME = 'event-bus'; - process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; const matcherConfig = [ { id: '1', @@ -43,7 +49,7 @@ describe('ConfigLoader Tests', () => { }, ]; runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(matcherConfig)); - vi.mocked(getParameter).mockResolvedValue('secret'); + githubWebhookSecretStore.get.mockResolvedValue('secret'); } it('should return the same instance of ConfigWebhook (singleton)', async () => { @@ -52,7 +58,7 @@ describe('ConfigLoader Tests', () => { const config2 = await ConfigWebhook.load(); expect(config1).toBe(config2); - expect(getParameter).toHaveBeenCalledOnce(); + expect(githubWebhookSecretStore.get).toHaveBeenCalledOnce(); expect(runnerMatcherConfigStore.get).toHaveBeenCalledOnce(); }); @@ -62,7 +68,7 @@ describe('ConfigLoader Tests', () => { const config2 = await ConfigWebhookEventBridge.load(); expect(config1).toBe(config2); - expect(getParameter).toHaveBeenCalledTimes(1); + expect(githubWebhookSecretStore.get).toHaveBeenCalledOnce(); expect(runnerMatcherConfigStore.get).not.toHaveBeenCalled(); }); @@ -72,7 +78,7 @@ describe('ConfigLoader Tests', () => { const config2 = await ConfigDispatcher.load(); expect(config1).toBe(config2); - expect(getParameter).not.toHaveBeenCalled(); + expect(githubWebhookSecretStore.get).not.toHaveBeenCalled(); expect(runnerMatcherConfigStore.get).toHaveBeenCalledOnce(); }); @@ -96,7 +102,6 @@ describe('ConfigLoader Tests', () => { describe('ConfigWebhook', () => { it('should load config successfully', async () => { process.env.REPOSITORY_ALLOW_LIST = '["repo1", "repo2"]'; - process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; const matcherConfig = [ { id: '1', @@ -108,7 +113,7 @@ describe('ConfigLoader Tests', () => { }, ]; runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(matcherConfig)); - vi.mocked(getParameter).mockResolvedValue('secret'); + githubWebhookSecretStore.get.mockResolvedValue('secret'); const config: ConfigWebhook = await ConfigWebhook.load(); @@ -118,7 +123,6 @@ describe('ConfigLoader Tests', () => { }); it('should load config successfully', async () => { - process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; const matcherConfig = [ { id: '1', @@ -130,7 +134,7 @@ describe('ConfigLoader Tests', () => { }, ]; runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(matcherConfig)); - vi.mocked(getParameter).mockResolvedValue('secret'); + githubWebhookSecretStore.get.mockResolvedValue('secret'); const config: ConfigWebhook = await ConfigWebhook.load(); @@ -146,7 +150,7 @@ describe('ConfigLoader Tests', () => { 'Failed to load parameter for matcherConfig from path /path/to/matcher/config: Failed to load matcher config', ), ); - vi.mocked(getParameter).mockResolvedValue(''); + githubWebhookSecretStore.get.mockResolvedValue(''); await expect(ConfigWebhook.load()).rejects.toThrow( 'Failed to load config: Failed to load parameter for matcherConfig from path /path/to/matcher/config: Failed to load matcher config', @@ -154,14 +158,12 @@ describe('ConfigLoader Tests', () => { }); it('should load combined matcher config returned by the store', async () => { - process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; - const combinedMatcherConfig = [ { id: '1', arn: 'arn:aws:sqs:queue1', matcherConfig: { labelMatchers: [['a']], exactMatch: true } }, { id: '2', arn: 'arn:aws:sqs:queue2', matcherConfig: { labelMatchers: [['b']], exactMatch: true } }, ]; runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(combinedMatcherConfig)); - vi.mocked(getParameter).mockResolvedValue('secret'); + githubWebhookSecretStore.get.mockResolvedValue('secret'); const config: ConfigWebhook = await ConfigWebhook.load(); @@ -170,13 +172,12 @@ describe('ConfigLoader Tests', () => { }); it('should propagate an error from the matcher config store', async () => { - process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; runnerMatcherConfigStore.get.mockRejectedValue( new Error( "Failed to load/parse combined matcher config: Expected ',' or ']' after array element in JSON at position 196", ), ); - vi.mocked(getParameter).mockResolvedValue('secret'); + githubWebhookSecretStore.get.mockResolvedValue('secret'); await expect(ConfigWebhook.load()).rejects.toThrow( "Failed to load config: Failed to load/parse combined matcher config: Expected ',' or ']' after array element in JSON at position 196", @@ -188,14 +189,7 @@ describe('ConfigLoader Tests', () => { it('should load config successfully', async () => { process.env.ACCEPT_EVENTS = '["push", "pull_request"]'; process.env.EVENT_BUS_NAME = 'event-bus'; - process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; - - vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { - if (paramPath === '/path/to/webhook/secret') { - return 'secret'; - } - return ''; - }); + githubWebhookSecretStore.get.mockResolvedValue('secret'); const config: ConfigWebhookEventBridge = await ConfigWebhookEventBridge.load(); @@ -206,13 +200,23 @@ describe('ConfigLoader Tests', () => { }); it('should throw error if config loading fails', async () => { - vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { - throw new Error(`Parameter ${paramPath} not found`); + githubWebhookSecretStore.get.mockRejectedValue(new Error('Webhook secret store is unavailable')); + + await expect(ConfigWebhookEventBridge.load()).rejects.toThrow( + 'Failed to load config: Environment variable for eventBusName is not set and no default value provided., Webhook secret store is unavailable', + ); + }); + + it('should report an error selecting the webhook secret store', async () => { + process.env.EVENT_BUS_NAME = 'event-bus'; + vi.mocked(getGitHubWebhookSecretStore).mockImplementationOnce(() => { + throw new Error("Unsupported runner config storage provider 'not-registered'"); }); await expect(ConfigWebhookEventBridge.load()).rejects.toThrow( - 'Failed to load config: Environment variable for eventBusName is not set and no default value provided., Failed to load parameter for webhookSecret from path undefined: Parameter undefined not found', + "Failed to load config: Unsupported runner config storage provider 'not-registered'", ); + expect(githubWebhookSecretStore.get).not.toHaveBeenCalled(); }); }); diff --git a/lambdas/functions/webhook/src/ConfigLoader.ts b/lambdas/functions/webhook/src/ConfigLoader.ts index 2cf261b849..e6d1d65004 100644 --- a/lambdas/functions/webhook/src/ConfigLoader.ts +++ b/lambdas/functions/webhook/src/ConfigLoader.ts @@ -1,10 +1,9 @@ -import { getParameter } from '@aws-github-runner/aws-ssm-util'; -import { getRunnerMatcherConfigStore } from '@aws-github-runner/storage-providers'; +import { getGitHubWebhookSecretStore, getRunnerMatcherConfigStore } from '@aws-github-runner/storage-providers'; import { RunnerMatcherConfig } from './sqs'; import { logger } from '@aws-github-runner/aws-powertools-util'; /** - * Base class for loading configuration from environment variables and SSM parameters. + * Base class for loading configuration from environment variables and configuration stores. * * @remarks * To avoid usages or checking values can be undefined we assume that configuration is @@ -55,16 +54,12 @@ abstract class BaseConfig { } } - protected async loadParameter(paramPath: string, propertyName: keyof this): Promise { - logger.debug(`Loading parameter for ${String(propertyName)} from path ${paramPath}`); - await getParameter(paramPath) - .then((value) => { - this.loadProperty(propertyName, value); - }) - .catch((error) => { - const errorMessage = `Failed to load parameter for ${String(propertyName)} from path ${paramPath}: ${(error as Error).message}`; - this.configLoadingErrors.push(errorMessage); - }); + protected async loadStoredProperty(propertyName: keyof this, getValue: () => Promise): Promise { + try { + this.loadProperty(propertyName, await getValue()); + } catch (error) { + this.configLoadingErrors.push((error as Error).message); + } } protected loadProperty(propertyName: keyof this, value: string) { @@ -117,7 +112,7 @@ export class ConfigWebhook extends MatcherAwareConfig { await Promise.all([ this.loadMatcherConfig(), - this.loadParameter(process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET, 'webhookSecret'), + this.loadStoredProperty('webhookSecret', () => getGitHubWebhookSecretStore().get()), ]); validateWebhookSecret(this); @@ -134,7 +129,7 @@ export class ConfigWebhookEventBridge extends BaseConfig { async loadConfig(): Promise { this.loadEnvVar(process.env.ACCEPT_EVENTS, 'allowedEvents', []); this.loadEnvVar(process.env.EVENT_BUS_NAME, 'eventBusName'); - await this.loadParameter(process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET, 'webhookSecret'); + await this.loadStoredProperty('webhookSecret', () => getGitHubWebhookSecretStore().get()); validateEventBusName(this); validateWebhookSecret(this); diff --git a/lambdas/functions/webhook/src/lambda.test.ts b/lambdas/functions/webhook/src/lambda.test.ts index 3bc67e42dd..b325c002f4 100644 --- a/lambdas/functions/webhook/src/lambda.test.ts +++ b/lambdas/functions/webhook/src/lambda.test.ts @@ -6,8 +6,12 @@ import { WorkflowJobEvent } from '@octokit/webhooks-types'; import { dispatchToRunners, eventBridgeWebhook, directWebhook } from './lambda'; import { publishForRunners, publishOnEventBridge } from './webhook'; import ValidationError from './ValidationError'; -import { getParameter } from '@aws-github-runner/aws-ssm-util'; -import { getRunnerMatcherConfigStore, type RunnerMatcherConfigStore } from '@aws-github-runner/storage-providers'; +import { + getGitHubWebhookSecretStore, + getRunnerMatcherConfigStore, + type GitHubWebhookSecretStore, + type RunnerMatcherConfigStore, +} from '@aws-github-runner/storage-providers'; import { dispatch } from './runners/dispatch'; import { EventWrapper } from './types'; import { describe, it, expect, beforeEach, vi } from 'vitest'; @@ -80,9 +84,11 @@ const context: Context = { vi.mock('./runners/dispatch'); vi.mock('./webhook'); -vi.mock('@aws-github-runner/aws-ssm-util'); vi.mock('@aws-github-runner/storage-providers'); +const githubWebhookSecretStore = { + get: vi.fn(), +} satisfies GitHubWebhookSecretStore; const runnerMatcherConfigStore = { get: vi.fn(), } satisfies RunnerMatcherConfigStore; @@ -92,7 +98,8 @@ describe('Test webhook lambda wrapper.', () => { vi.clearAllMocks(); // The handlers only need non-empty config values because their downstream // implementations are mocked in this wrapper test. - vi.mocked(getParameter).mockResolvedValue('["abc"]'); + vi.mocked(getGitHubWebhookSecretStore).mockReturnValue(githubWebhookSecretStore); + githubWebhookSecretStore.get.mockResolvedValue('["abc"]'); vi.mocked(getRunnerMatcherConfigStore).mockReturnValue(runnerMatcherConfigStore); runnerMatcherConfigStore.get.mockResolvedValue('["abc"]'); }); diff --git a/lambdas/functions/webhook/src/modules.d.ts b/lambdas/functions/webhook/src/modules.d.ts index 3b04a2a5be..05a81a12ab 100644 --- a/lambdas/functions/webhook/src/modules.d.ts +++ b/lambdas/functions/webhook/src/modules.d.ts @@ -2,7 +2,6 @@ declare namespace NodeJS { export interface ProcessEnv { ENVIRONMENT: string; EVENT_BUS_NAME: string; - PARAMETER_GITHUB_APP_WEBHOOK_SECRET: string; QUEUE_SELECTION_STRATEGY: string; REPOSITORY_ALLOW_LIST: string; RUNNER_LABELS: string; diff --git a/lambdas/functions/webhook/src/webhook/index.test.ts b/lambdas/functions/webhook/src/webhook/index.test.ts index 43345388f7..6d7a272309 100644 --- a/lambdas/functions/webhook/src/webhook/index.test.ts +++ b/lambdas/functions/webhook/src/webhook/index.test.ts @@ -1,6 +1,10 @@ import { Webhooks } from '@octokit/webhooks'; -import { getParameter } from '@aws-github-runner/aws-ssm-util'; -import { getRunnerMatcherConfigStore, type RunnerMatcherConfigStore } from '@aws-github-runner/storage-providers'; +import { + getGitHubWebhookSecretStore, + getRunnerMatcherConfigStore, + type GitHubWebhookSecretStore, + type RunnerMatcherConfigStore, +} from '@aws-github-runner/storage-providers'; import nock from 'nock'; @@ -16,10 +20,12 @@ import { describe, it, expect, beforeEach, vi } from 'vitest'; vi.mock('../sqs'); vi.mock('../eventbridge'); vi.mock('../runners/dispatch'); -vi.mock('@aws-github-runner/aws-ssm-util'); vi.mock('@aws-github-runner/storage-providers'); const GITHUB_APP_WEBHOOK_SECRET = 'TEST_SECRET'; +const githubWebhookSecretStore = { + get: vi.fn(), +} satisfies GitHubWebhookSecretStore; const runnerMatcherConfigStore = { get: vi.fn(), } satisfies RunnerMatcherConfigStore; @@ -290,7 +296,6 @@ describe('Check message size (checkBodySize)', () => { }); function mockConfigResponse() { - process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; const matcherConfig = [ { id: '1', @@ -301,7 +306,8 @@ function mockConfigResponse() { }, }, ]; + vi.mocked(getGitHubWebhookSecretStore).mockReturnValue(githubWebhookSecretStore); vi.mocked(getRunnerMatcherConfigStore).mockReturnValue(runnerMatcherConfigStore); + githubWebhookSecretStore.get.mockResolvedValue(GITHUB_APP_WEBHOOK_SECRET); runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(matcherConfig)); - vi.mocked(getParameter).mockResolvedValue(GITHUB_APP_WEBHOOK_SECRET); } diff --git a/lambdas/libs/storage-providers/aws/ssm/environment.d.ts b/lambdas/libs/storage-providers/aws/ssm/environment.d.ts index 5ffa7b85bb..ab8b54e88a 100644 --- a/lambdas/libs/storage-providers/aws/ssm/environment.d.ts +++ b/lambdas/libs/storage-providers/aws/ssm/environment.d.ts @@ -3,6 +3,7 @@ export {}; declare global { namespace NodeJS { interface ProcessEnv { + PARAMETER_GITHUB_APP_WEBHOOK_SECRET?: string; PARAMETER_RUNNER_MATCHER_CONFIG_PATH?: string; SSM_PARAMETER_STORE_TAGS?: string; SSM_CONFIG_PATH?: string; diff --git a/lambdas/libs/storage-providers/aws/ssm/github-webhook-secret-store.test.ts b/lambdas/libs/storage-providers/aws/ssm/github-webhook-secret-store.test.ts new file mode 100644 index 0000000000..7384c769bc --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm/github-webhook-secret-store.test.ts @@ -0,0 +1,51 @@ +import { getParameter } from '@aws-github-runner/aws-ssm-util'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAwsSsmGitHubWebhookSecretStore } from './github-webhook-secret-store'; + +vi.mock('@aws-github-runner/aws-ssm-util', () => ({ + getParameter: vi.fn(), +})); + +const getParameterMock = vi.mocked(getParameter); +const cleanEnv = process.env; +const webhookSecretParameter = '/actions-runner/test/webhook_secret'; + +describe('aws_ssm GitHub webhook secret store', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env = { ...cleanEnv }; + process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = webhookSecretParameter; + }); + + it('loads the webhook secret parameter', async () => { + getParameterMock.mockResolvedValue('fake-webhook-secret'); + const store = createAwsSsmGitHubWebhookSecretStore(); + + await expect(store.get()).resolves.toBe('fake-webhook-secret'); + expect(getParameterMock).toHaveBeenCalledOnce(); + expect(getParameterMock).toHaveBeenCalledWith(webhookSecretParameter); + }); + + it('wraps a parameter read failure with the legacy error message', async () => { + getParameterMock.mockRejectedValue(new Error('access denied')); + const store = createAwsSsmGitHubWebhookSecretStore(); + + await expect(store.get()).rejects.toThrow( + `Failed to load parameter for webhookSecret from path ${webhookSecretParameter}: access denied`, + ); + }); + + it.each([undefined, '', ' '])('requires a webhook secret parameter path for input %j', (parameterPath) => { + if (parameterPath === undefined) { + delete process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET; + } else { + process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = parameterPath; + } + + expect(() => createAwsSsmGitHubWebhookSecretStore()).toThrow( + 'Environment variable PARAMETER_GITHUB_APP_WEBHOOK_SECRET is not set', + ); + expect(getParameterMock).not.toHaveBeenCalled(); + }); +}); diff --git a/lambdas/libs/storage-providers/aws/ssm/github-webhook-secret-store.ts b/lambdas/libs/storage-providers/aws/ssm/github-webhook-secret-store.ts new file mode 100644 index 0000000000..ce35e1f532 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm/github-webhook-secret-store.ts @@ -0,0 +1,27 @@ +import { getParameter } from '@aws-github-runner/aws-ssm-util'; + +import type { GitHubWebhookSecretStore } from '../../core'; +import type {} from './environment'; + +export function createAwsSsmGitHubWebhookSecretStore(): GitHubWebhookSecretStore { + const parameterPath = process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET; + if (!parameterPath || parameterPath.trim() === '') { + throw new Error('Environment variable PARAMETER_GITHUB_APP_WEBHOOK_SECRET is not set'); + } + + return new AwsSsmGitHubWebhookSecretStore(parameterPath); +} + +class AwsSsmGitHubWebhookSecretStore implements GitHubWebhookSecretStore { + constructor(private readonly parameterPath: string) {} + + async get(): Promise { + try { + return await getParameter(this.parameterPath); + } catch (error) { + throw new Error( + `Failed to load parameter for webhookSecret from path ${this.parameterPath}: ${(error as Error).message}`, + ); + } + } +} diff --git a/lambdas/libs/storage-providers/core/index.ts b/lambdas/libs/storage-providers/core/index.ts index 3d9fc43fe8..e6060eae2a 100644 --- a/lambdas/libs/storage-providers/core/index.ts +++ b/lambdas/libs/storage-providers/core/index.ts @@ -1,3 +1,6 @@ +export interface GitHubWebhookSecretStore { + get(): Promise; +} export interface RunnerConfigMetadata { key: string; value: string; diff --git a/lambdas/libs/storage-providers/github-webhook-secret.test.ts b/lambdas/libs/storage-providers/github-webhook-secret.test.ts new file mode 100644 index 0000000000..5888e735b0 --- /dev/null +++ b/lambdas/libs/storage-providers/github-webhook-secret.test.ts @@ -0,0 +1,83 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAwsSsmGitHubWebhookSecretStore } from './aws/ssm/github-webhook-secret-store'; +import type { GitHubWebhookSecretStore } from './core'; +import { getGitHubWebhookSecretStore, resetGitHubWebhookSecretStore } from './github-webhook-secret'; + +vi.mock('./aws/ssm/github-webhook-secret-store', () => ({ + createAwsSsmGitHubWebhookSecretStore: vi.fn(), +})); + +const createAwsSsmGitHubWebhookSecretStoreMock = vi.mocked(createAwsSsmGitHubWebhookSecretStore); +const cleanEnv = process.env; + +describe('GitHub webhook secret store selection', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env = { ...cleanEnv }; + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + resetGitHubWebhookSecretStore(); + }); + + it.each([undefined, '', ' '])('uses aws_ssm for default selector input %j', (provider) => { + setProvider(provider); + const store = stubStore(); + + expect(getGitHubWebhookSecretStore()).toBe(store); + expect(createAwsSsmGitHubWebhookSecretStoreMock).toHaveBeenCalledOnce(); + }); + + it.each(['aws_ssm', ' AWS_SSM '])('uses aws_ssm for explicit selector input %j', (provider) => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + const store = stubStore(); + + expect(getGitHubWebhookSecretStore()).toBe(store); + expect(createAwsSsmGitHubWebhookSecretStoreMock).toHaveBeenCalledOnce(); + }); + + it('rejects an unsupported provider on first use', () => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = 'not-registered'; + + expect(() => getGitHubWebhookSecretStore()).toThrow("Unsupported runner config storage provider 'not-registered'"); + expect(createAwsSsmGitHubWebhookSecretStoreMock).not.toHaveBeenCalled(); + }); + + it('selects lazily and caches the created store', () => { + const store = stubStore(); + + expect(createAwsSsmGitHubWebhookSecretStoreMock).not.toHaveBeenCalled(); + const first = getGitHubWebhookSecretStore(); + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = 'not-registered'; + const second = getGitHubWebhookSecretStore(); + + expect(first).toBe(store); + expect(second).toBe(store); + expect(createAwsSsmGitHubWebhookSecretStoreMock).toHaveBeenCalledOnce(); + }); + + it('selects again after the test reset', () => { + const firstStore = stubStore(); + expect(getGitHubWebhookSecretStore()).toBe(firstStore); + + const secondStore = { get: vi.fn() } satisfies GitHubWebhookSecretStore; + createAwsSsmGitHubWebhookSecretStoreMock.mockReturnValue(secondStore); + resetGitHubWebhookSecretStore(); + + expect(getGitHubWebhookSecretStore()).toBe(secondStore); + expect(createAwsSsmGitHubWebhookSecretStoreMock).toHaveBeenCalledTimes(2); + }); +}); + +function setProvider(provider: string | undefined): void { + if (provider === undefined) { + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + } else { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + } +} + +function stubStore(): GitHubWebhookSecretStore { + const store = { get: vi.fn() } satisfies GitHubWebhookSecretStore; + createAwsSsmGitHubWebhookSecretStoreMock.mockReturnValue(store); + return store; +} diff --git a/lambdas/libs/storage-providers/github-webhook-secret.ts b/lambdas/libs/storage-providers/github-webhook-secret.ts new file mode 100644 index 0000000000..f13df08718 --- /dev/null +++ b/lambdas/libs/storage-providers/github-webhook-secret.ts @@ -0,0 +1,23 @@ +import { createAwsSsmGitHubWebhookSecretStore } from './aws/ssm/github-webhook-secret-store'; +import type { GitHubWebhookSecretStore } from './core'; +import type {} from './environment'; +import { resolveRunnerConfigStorageProvider, type RunnerConfigStorageProvider } from './provider'; + +type GitHubWebhookSecretStoreFactory = () => GitHubWebhookSecretStore; + +const providerFactories = { + aws_ssm: createAwsSsmGitHubWebhookSecretStore, +} as const satisfies Record; + +let githubWebhookSecretStore: GitHubWebhookSecretStore | undefined; + +export function getGitHubWebhookSecretStore(): GitHubWebhookSecretStore { + githubWebhookSecretStore ??= + providerFactories[resolveRunnerConfigStorageProvider(process.env.RUNNER_CONFIG_STORAGE_PROVIDER)](); + return githubWebhookSecretStore; +} + +// Test-only reset for cases that need to exercise first-use environment selection. +export function resetGitHubWebhookSecretStore(): void { + githubWebhookSecretStore = undefined; +} diff --git a/lambdas/libs/storage-providers/index.ts b/lambdas/libs/storage-providers/index.ts index 026b79d48a..fcd4bd9338 100644 --- a/lambdas/libs/storage-providers/index.ts +++ b/lambdas/libs/storage-providers/index.ts @@ -1,6 +1,7 @@ export type { GitHubAppCredential, GitHubAppCredentialsStore, + GitHubWebhookSecretStore, RunnerConfigConsumer, RunnerConfigConsumeOptions, RunnerConfigHousekeeper, @@ -21,4 +22,5 @@ export { export type { RunnerConfigStorageProvider } from './provider'; export { createCommonStorage, createStorageProviders } from './storage-providers'; export type { StorageProviders, RunnerConfigStorage, CommonStorage } from './core'; +export { getGitHubWebhookSecretStore, resetGitHubWebhookSecretStore } from './github-webhook-secret'; export { getRunnerMatcherConfigStore, resetRunnerMatcherConfigStore } from './runner-matcher-config'; diff --git a/lambdas/libs/storage-providers/vitest.config.ts b/lambdas/libs/storage-providers/vitest.config.ts index f107f07411..a009dab1bf 100644 --- a/lambdas/libs/storage-providers/vitest.config.ts +++ b/lambdas/libs/storage-providers/vitest.config.ts @@ -13,6 +13,7 @@ export default mergeConfig(defaultConfig, { 'runner-config-consumer.ts', 'storage-providers.ts', 'provider.ts', + 'github-webhook-secret.ts', 'runner-matcher-config.ts', 'core/**/*.ts', 'aws/**/*.ts', diff --git a/lambdas/yarn.lock b/lambdas/yarn.lock index 49dbdd1d66..17e99e638d 100644 --- a/lambdas/yarn.lock +++ b/lambdas/yarn.lock @@ -241,7 +241,6 @@ __metadata: resolution: "@aws-github-runner/webhook@workspace:functions/webhook" dependencies: "@aws-github-runner/aws-powertools-util": "npm:*" - "@aws-github-runner/aws-ssm-util": "npm:*" "@aws-github-runner/compute-providers": "npm:*" "@aws-github-runner/storage-providers": "npm:*" "@aws-sdk/client-eventbridge": "npm:^3.1009.0" From 045a51880782df68fd2a289dacc7ef88b2af7c20 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 8 Sep 2026 23:22:32 +0200 Subject: [PATCH 3/3] feat(storage): add shared DynamoDB runner storage --- .../src/pool/pool-contract.test.ts | 7 + .../functions/control-plane/src/pool/pool.ts | 69 ++- .../src/scale-runners/github-runner.ts | 195 +++++- .../src/scale-runners/job-retry.test.ts | 7 + .../scale-runners/scale-down-contract.test.ts | 4 + .../src/scale-runners/scale-down.test.ts | 353 +++++++++++ .../src/scale-runners/scale-down.ts | 389 ++++++++++-- .../scale-runners/scale-up-contract.test.ts | 9 + .../src/scale-runners/scale-up.test.ts | 4 +- .../src/scale-runners/scale-up.ts | 39 +- .../compute-provider-contracts/scale-up.ts | 4 +- .../ec2/src/control-plane/runner-creation.ts | 3 + .../ec2/src/control-plane/scale-up.test.ts | 2 +- .../aws/ec2/src/control-plane/scale-up.ts | 4 +- .../aws/ec2/src/environment.d.ts | 1 + lambdas/libs/compute-providers/core/index.ts | 9 +- .../storage-providers/aws/dynamodb/client.ts | 21 + .../aws/dynamodb/durable-config.ts | 37 ++ .../aws/dynamodb/environment.d.ts | 13 + .../aws/dynamodb/environment.ts | 29 + .../github-app-credentials-store.test.ts | 103 ++++ .../dynamodb/github-app-credentials-store.ts | 89 +++ .../github-webhook-secret-store.test.ts | 74 +++ .../dynamodb/github-webhook-secret-store.ts | 21 + .../aws/dynamodb/keys.test.ts | 50 ++ .../storage-providers/aws/dynamodb/keys.ts | 34 ++ .../aws/dynamodb/runner-config-store.test.ts | 148 +++++ .../aws/dynamodb/runner-config-store.ts | 66 +++ .../dynamodb/runner-group-cache-store.test.ts | 108 ++++ .../aws/dynamodb/runner-group-cache-store.ts | 80 +++ .../runner-matcher-config-store.test.ts | 95 +++ .../dynamodb/runner-matcher-config-store.ts | 27 + .../aws/dynamodb/runner-state-store.test.ts | 451 ++++++++++++++ .../aws/dynamodb/runner-state-store.ts | 556 ++++++++++++++++++ lambdas/libs/storage-providers/core/index.ts | 51 ++ .../github-app-credentials.test.ts | 97 +++ .../github-app-credentials.ts | 25 + .../github-webhook-secret.test.ts | 56 +- .../github-webhook-secret.ts | 2 + lambdas/libs/storage-providers/index.ts | 12 + lambdas/libs/storage-providers/package.json | 1 + lambdas/libs/storage-providers/provider.ts | 6 +- .../storage-providers/runner-config.test.ts | 108 ++++ .../libs/storage-providers/runner-config.ts | 25 + .../runner-group-cache.test.ts | 106 ++++ .../storage-providers/runner-group-cache.ts | 25 + .../runner-matcher-config.test.ts | 23 + .../runner-matcher-config.ts | 2 + .../storage-providers/runner-state.test.ts | 89 +++ .../libs/storage-providers/runner-state.ts | 25 + .../libs/storage-providers/vitest.config.ts | 1 + lambdas/yarn.lock | 342 +++++++++++ 52 files changed, 3999 insertions(+), 98 deletions(-) create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/client.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/durable-config.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/environment.d.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/environment.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/github-app-credentials-store.test.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/github-app-credentials-store.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/github-webhook-secret-store.test.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/github-webhook-secret-store.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/keys.test.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/keys.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/runner-config-store.test.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/runner-config-store.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/runner-group-cache-store.test.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/runner-group-cache-store.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/runner-matcher-config-store.test.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/runner-matcher-config-store.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/runner-state-store.test.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/runner-state-store.ts create mode 100644 lambdas/libs/storage-providers/github-app-credentials.test.ts create mode 100644 lambdas/libs/storage-providers/github-app-credentials.ts create mode 100644 lambdas/libs/storage-providers/runner-config.test.ts create mode 100644 lambdas/libs/storage-providers/runner-config.ts create mode 100644 lambdas/libs/storage-providers/runner-group-cache.test.ts create mode 100644 lambdas/libs/storage-providers/runner-group-cache.ts create mode 100644 lambdas/libs/storage-providers/runner-state.test.ts create mode 100644 lambdas/libs/storage-providers/runner-state.ts diff --git a/lambdas/functions/control-plane/src/pool/pool-contract.test.ts b/lambdas/functions/control-plane/src/pool/pool-contract.test.ts index 1a208edde7..4f9628fd05 100644 --- a/lambdas/functions/control-plane/src/pool/pool-contract.test.ts +++ b/lambdas/functions/control-plane/src/pool/pool-contract.test.ts @@ -17,6 +17,13 @@ vi.mock('../github/auth', () => ({ getStoredInstallationId: vi.fn().mockResolvedValue(undefined), })); +vi.mock('@aws-github-runner/storage-providers', () => ({ + createStorageProviders: vi.fn().mockReturnValue({ + githubAppCredentials: { get: vi.fn() }, + }), + getRunnerStateStore: vi.fn().mockReturnValue(undefined), +})); + vi.mock('../scale-runners/github-runner', () => ({ createStartRunnerConfig: vi.fn(), getGitHubEnterpriseApiUrl: vi.fn(), diff --git a/lambdas/functions/control-plane/src/pool/pool.ts b/lambdas/functions/control-plane/src/pool/pool.ts index 9f91c6cbcd..60c7bc2646 100644 --- a/lambdas/functions/control-plane/src/pool/pool.ts +++ b/lambdas/functions/control-plane/src/pool/pool.ts @@ -1,7 +1,12 @@ import { Octokit } from '@octokit/rest'; import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; import { resolveComputeProviderType } from '@aws-github-runner/compute-providers/provider-types'; -import { createStorageProviders, type StorageProviders } from '@aws-github-runner/storage-providers'; +import { + createStorageProviders, + getRunnerStateStore, + type RunnerStateRecord, + type StorageProviders, +} from '@aws-github-runner/storage-providers'; import yn from 'yn'; import { @@ -60,26 +65,43 @@ export async function adjust(event: PoolEvent): Promise { runnerNamePrefix, ); - // Look up the managed provider runners, but running does not mean idle. - const poolRunners = await computeProvider.listRunners({ + const runnerStateStore = getRunnerStateStore(); + let currentRunnerCount: number; + let numberOfRunnersInPool: number; + const providerRunners = await computeProvider.listRunners({ environment, runnerOwner, runnerType: 'Org', }); - - const numberOfRunnersInPool = computeProvider.countAvailableRunners(poolRunners, runnerStatusses, includeBusyRunners); + if (runnerStateStore) { + const runnerStates = (await runnerStateStore.list({ computeProvider: computeProvider.type })).filter( + (record) => record.runnerOwner === runnerOwner && record.runnerType === 'Org', + ); + const storedComputeResourceIds = new Set(runnerStates.map((record) => record.computeResourceId)); + const untrackedProviderRunners = providerRunners.filter((runner) => !storedComputeResourceIds.has(runner.id)); + // Inventory is canonical for tracked resources. Provider discovery contributes + // only untracked resources so a launch-before-state crash cannot over-provision. + currentRunnerCount = runnerStates.length + untrackedProviderRunners.length; + numberOfRunnersInPool = + countAvailableStoredRunners(runnerStates, runnerStatusses, includeBusyRunners) + + computeProvider.countAvailableRunners(untrackedProviderRunners, runnerStatusses, includeBusyRunners); + } else { + // Look up the managed provider runners, but running does not mean idle. + currentRunnerCount = providerRunners.length; + numberOfRunnersInPool = computeProvider.countAvailableRunners(providerRunners, runnerStatusses, includeBusyRunners); + } let topUp = event.poolSize - numberOfRunnersInPool; // The pool must never push the total number of runners (busy + idle) past the configured maximum. - // poolRunners contains every running runner for this type, so its length is the current total and no - // extra API call is needed. Without this clamp the pool keeps topping up against idle-only counts and - // can overshoot runners_maximum_count, while the scale-up lambda correctly refuses to launch. + // currentRunnerCount includes both canonical inventory and untracked provider recovery records. Without + // this clamp the pool keeps topping up against idle-only counts and can overshoot runners_maximum_count, + // while the scale-up lambda correctly refuses to launch. if (maximumRunners !== -1 && topUp > 0) { - const headroom = maximumRunners - poolRunners.length; + const headroom = maximumRunners - currentRunnerCount; if (topUp > headroom) { logger.info( `Capping pool top-up from ${topUp} to ${Math.max(headroom, 0)} to respect the maximum of ` + - `${maximumRunners} runners (currently ${poolRunners.length} running).`, + `${maximumRunners} runners (currently ${currentRunnerCount} running).`, ); topUp = headroom; } @@ -129,6 +151,33 @@ async function getInstallationId( ).data.id; } +function countAvailableStoredRunners( + runnerStates: RunnerStateRecord[], + runnerStatuses: Map, + includeBusyRunners: boolean, +): number { + let available = 0; + for (const runner of runnerStates) { + if (runner.state === 'orphan' || runner.state === 'terminating') { + continue; + } + + const status = runnerStatuses.get(runner.computeResourceId); + if ((status?.busy === false || includeBusyRunners) && status?.status === 'online') { + available++; + } else if (status === undefined && !runnerBootTimeExceeded(runner.createdAt)) { + available++; + } + } + return available; +} + +function runnerBootTimeExceeded(createdAt: string): boolean { + const bootTimeMinutes = Number(process.env.RUNNER_BOOT_TIME_IN_MINUTES); + const launchTime = new Date(createdAt).getTime(); + return launchTime + bootTimeMinutes * 60_000 < Date.now(); +} + async function getGitHubRegisteredRunnnerStatusses( ghClient: Octokit, runnerOwner: string, diff --git a/lambdas/functions/control-plane/src/scale-runners/github-runner.ts b/lambdas/functions/control-plane/src/scale-runners/github-runner.ts index b344012149..b697ba5884 100644 --- a/lambdas/functions/control-plane/src/scale-runners/github-runner.ts +++ b/lambdas/functions/control-plane/src/scale-runners/github-runner.ts @@ -1,11 +1,16 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; import { createStorageProviders, + getRunnerConfigStore, + getRunnerStateStore, type RunnerConfigMetadata, + type RunnerConfigRecord, type RunnerConfigStore, type GitHubAppCredentialsStore, type RunnerGroupCacheStore, + type RunnerStateStore, } from '@aws-github-runner/storage-providers'; +import type { ComputeProviderType } from '@aws-github-runner/compute-providers/provider-types'; import { Octokit } from '@octokit/rest'; import { getStoredInstallationId } from '../github/auth'; @@ -22,6 +27,8 @@ export interface GitHubRunnerMetadata { export interface StartRunnerConfigOptions { runnerConfigStore?: RunnerConfigStore; runnerGroupCacheStore?: RunnerGroupCacheStore; + computeProvider?: ComputeProviderType; + getRunnerConfigAccessScope?: (runnerId: string) => string | undefined; getRunnerConfigMetadata?: (runnerId: string) => RunnerConfigMetadata[]; onJitConfigCreated?: (runnerId: string, metadata: GitHubRunnerMetadata) => Promise; } @@ -207,11 +214,22 @@ export async function createStartRunnerConfig( ghClient: Octokit, options: StartRunnerConfigOptions = {}, ): Promise { - const runnerConfigStore = options.runnerConfigStore ?? createStorageProviders().runnerConfig; + const runnerConfigStore = options.runnerConfigStore ?? getRunnerConfigStore(); + const runnerStateStore = getRunnerStateStore(); + if (runnerStateStore && options.computeProvider === undefined) { + throw new Error('A compute provider is required when runner state storage is enabled'); + } if (githubRunnerConfig.enableJitConfig && githubRunnerConfig.ephemeral) { - return await createJitConfig(githubRunnerConfig, runnerIds, ghClient, runnerConfigStore, options); + return await createJitConfig(githubRunnerConfig, runnerIds, ghClient, runnerConfigStore, runnerStateStore, options); } else { - return await createRegistrationTokenConfig(githubRunnerConfig, runnerIds, ghClient, runnerConfigStore, options); + return await createRegistrationTokenConfig( + githubRunnerConfig, + runnerIds, + ghClient, + runnerConfigStore, + runnerStateStore, + options, + ); } } @@ -226,13 +244,15 @@ function addDelay(runnerIds: string[], runnerConfigStore: RunnerConfigStore) { /** * Creates registration token configuration for non-ephemeral runners. * - * @returns Empty array (this configuration method does not have failure cases) + * @returns Runner IDs whose durable configuration failed. The legacy SSM path + * still throws on the first failure so existing all-or-nothing cleanup is preserved. */ async function createRegistrationTokenConfig( githubRunnerConfig: CreateGitHubRunnerConfig, runnerIds: string[], ghClient: Octokit, runnerConfigStore: RunnerConfigStore, + runnerStateStore: RunnerStateStore | undefined, options: StartRunnerConfigOptions, ): Promise { const { isDelay, delay, delayMilliseconds } = addDelay(runnerIds, runnerConfigStore); @@ -243,18 +263,53 @@ async function createRegistrationTokenConfig( runner_service_config: removeTokenFromLogging(runnerServiceConfig), }); + if (!runnerStateStore) { + for (const runnerId of runnerIds) { + const metadata = options.getRunnerConfigMetadata?.(runnerId); + await runnerConfigStore.create(createRunnerConfigRecord(runnerId, runnerServiceConfig.join(' '), options), { + metadata, + }); + if (isDelay) { + // Delay to stay within the selected store's maximum write throughput. + await delay(delayMilliseconds); + } + } + return []; + } + + const failedRunnerIds: string[] = []; for (const runnerId of runnerIds) { - await runnerConfigStore.create( - { runnerId, value: runnerServiceConfig.join(' ') }, - { metadata: options.getRunnerConfigMetadata?.(runnerId) }, - ); - if (isDelay) { - // Delay to stay within the selected store's maximum write throughput. - await delay(delayMilliseconds); + try { + const metadata = options.getRunnerConfigMetadata?.(runnerId); + await createProvisioningRunnerState(runnerStateStore, githubRunnerConfig, runnerId, options, metadata); + await runnerConfigStore.create(createRunnerConfigRecord(runnerId, runnerServiceConfig.join(' '), options), { + metadata, + }); + await runnerStateStore.activate(runnerId, { metadata }); + if (isDelay) { + // Delay to stay within the selected store's maximum write throughput. + await delay(delayMilliseconds); + } + } catch (error) { + failedRunnerIds.push(runnerId); + logger.warn('Failed to configure runner, continuing with remaining instances', { + instance: runnerId, + error: error instanceof Error ? error.message : String(error), + retryable: true, + }); } } - return []; + if (failedRunnerIds.length > 0) { + logger.error('Failed to configure some runner instances', { + failedInstances: failedRunnerIds, + totalInstances: runnerIds.length, + successfulInstances: runnerIds.length - failedRunnerIds.length, + retryable: true, + }); + } + + return failedRunnerIds; } /** @@ -268,6 +323,7 @@ async function createJitConfig( runnerIds: string[], ghClient: Octokit, runnerConfigStore: RunnerConfigStore, + runnerStateStore: RunnerStateStore | undefined, options: StartRunnerConfigOptions, ): Promise { const runnerGroupId = await getRunnerGroupId(githubRunnerConfig, ghClient, options.runnerGroupCacheStore); @@ -285,6 +341,17 @@ async function createJitConfig( runnerGroupId: runnerGroupId, runnerLabels: runnerLabels, }; + // Durable inventory needs metadata before JIT creation so failures leave a + // complete provisioning record. The legacy path retains its original timing. + let metadata = runnerStateStore ? options.getRunnerConfigMetadata?.(runnerId) : undefined; + await createProvisioningRunnerState( + runnerStateStore, + githubRunnerConfig, + runnerId, + options, + metadata, + ephemeralRunnerConfig, + ); logger.debug(`Runner name: ${ephemeralRunnerConfig.runnerName}`); const runnerConfig = githubRunnerConfig.runnerType === 'Org' @@ -304,18 +371,47 @@ async function createJitConfig( metricGitHubAppRateLimit(runnerConfig.headers, githubRunnerConfig.appIndex); - await options.onJitConfigCreated?.(runnerId, { + const githubRunnerMetadata = { githubRunnerId: runnerConfig.data.runner.id.toString(), runnerLabels, - }); + }; + if (runnerStateStore) { + try { + await runnerStateStore.recordGitHubIdentity(runnerId, { + ...githubRunnerMetadata, + runnerName: ephemeralRunnerConfig.runnerName, + metadata, + }); + } catch (error) { + await deregisterJitRunnerAfterIdentityWriteFailure( + githubRunnerConfig, + ghClient, + runnerConfig.data.runner.id, + runnerId, + ); + throw error; + } + } + await options.onJitConfigCreated?.(runnerId, githubRunnerMetadata); logger.debug('Runner JIT config for ephemeral runner generated.', { instance: runnerId, }); + if (!runnerStateStore) { + metadata = options.getRunnerConfigMetadata?.(runnerId); + } await runnerConfigStore.create( - { runnerId, value: runnerConfig.data.encoded_jit_config }, - { metadata: options.getRunnerConfigMetadata?.(runnerId) }, + createRunnerConfigRecord(runnerId, runnerConfig.data.encoded_jit_config, options), + { + metadata, + }, ); + await runnerStateStore?.activate(runnerId, { + githubRunnerId: githubRunnerMetadata.githubRunnerId, + runnerLabels, + runnerName: ephemeralRunnerConfig.runnerName, + metadata, + }); if (isDelay) { // Delay to stay within the selected store's maximum write throughput. await delay(delayMilliseconds); @@ -342,6 +438,73 @@ async function createJitConfig( return failedRunnerIds; } +async function deregisterJitRunnerAfterIdentityWriteFailure( + githubRunnerConfig: CreateGitHubRunnerConfig, + ghClient: Octokit, + githubRunnerId: number, + runnerId: string, +): Promise { + try { + if (githubRunnerConfig.runnerType === 'Org') { + await ghClient.actions.deleteSelfHostedRunnerFromOrg({ + org: githubRunnerConfig.runnerOwner, + runner_id: githubRunnerId, + }); + } else { + const [owner, repo] = githubRunnerConfig.runnerOwner.split('/'); + await ghClient.actions.deleteSelfHostedRunnerFromRepo({ owner, repo, runner_id: githubRunnerId }); + } + logger.info('De-registered JIT GitHub runner after its inventory identity could not be stored', { + instance: runnerId, + githubRunnerId, + }); + } catch (cleanupError) { + logger.error('Failed to de-register JIT GitHub runner after its inventory identity could not be stored', { + instance: runnerId, + githubRunnerId, + error: cleanupError instanceof Error ? cleanupError.message : String(cleanupError), + retryable: true, + }); + } +} + +function createRunnerConfigRecord( + runnerId: string, + value: string, + options: StartRunnerConfigOptions, +): RunnerConfigRecord { + const accessScope = options.getRunnerConfigAccessScope?.(runnerId); + return { + runnerId, + value, + ...(accessScope === undefined ? {} : { accessScope }), + }; +} + +async function createProvisioningRunnerState( + runnerStateStore: RunnerStateStore | undefined, + githubRunnerConfig: CreateGitHubRunnerConfig, + runnerId: string, + options: StartRunnerConfigOptions, + metadata: RunnerConfigMetadata[] | undefined, + jitConfig?: EphemeralRunnerConfig, +): Promise { + if (!runnerStateStore) { + return; + } + + await runnerStateStore.create({ + runnerId, + computeProvider: options.computeProvider!, + computeResourceId: runnerId, + runnerOwner: githubRunnerConfig.runnerOwner, + runnerType: githubRunnerConfig.runnerType, + runnerName: jitConfig?.runnerName, + runnerLabels: jitConfig?.runnerLabels, + metadata, + }); +} + export function getGitHubEnterpriseApiUrl() { const ghesBaseUrl = process.env.GHES_URL; let ghesApiUrl = ''; diff --git a/lambdas/functions/control-plane/src/scale-runners/job-retry.test.ts b/lambdas/functions/control-plane/src/scale-runners/job-retry.test.ts index c4ff1e5d76..dd1cd503ea 100644 --- a/lambdas/functions/control-plane/src/scale-runners/job-retry.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/job-retry.test.ts @@ -12,6 +12,13 @@ vi.mock('../aws/sqs', async () => ({ publishMessage: vi.fn(), })); +vi.mock('@aws-github-runner/storage-providers', () => ({ + getGitHubAppCredentialsStore: vi.fn(), + getRunnerConfigStore: vi.fn().mockReturnValue({ houseKeeper: vi.fn() }), + getRunnerGroupCacheStore: vi.fn(), + getRunnerStateStore: vi.fn().mockReturnValue(undefined), +})); + vi.mock('@aws-github-runner/aws-powertools-util', async () => { // This is a workaround for TypeScript's type checking // Use vi.importActual with a type assertion to avoid spread operator type error diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-contract.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-contract.test.ts index a4d52acc40..154a93bed9 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down-contract.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-contract.test.ts @@ -7,6 +7,10 @@ import { controlPlaneProviderRegistry } from '../control-plane-providers'; import { scaleDown } from './scale-down'; import type { ScaleDownComputeProvider } from './types'; +vi.mock('@aws-github-runner/storage-providers', () => ({ + getRunnerStateStore: vi.fn().mockReturnValue(undefined), +})); + const mockedResolveCapability = vi.spyOn(controlPlaneProviderRegistry, 'capability'); const cleanEnv = process.env; diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts index 3583247f8d..e41b0665a5 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts @@ -1,5 +1,10 @@ import type { Octokit } from '@octokit/rest'; import { RequestError } from '@octokit/request-error'; +import { + getRunnerStateStore, + type RunnerStateRecord, + type RunnerStateStore, +} from '@aws-github-runner/storage-providers'; import moment from 'moment'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { defaultComputeProvider } from '@aws-github-runner/compute-providers/provider-types'; @@ -17,6 +22,10 @@ vi.mock('../github/auth', () => ({ getStoredInstallationId: vi.fn().mockResolvedValue(undefined), })); +vi.mock('@aws-github-runner/storage-providers', () => ({ + getRunnerStateStore: vi.fn(), +})); + const mockOctokit = { apps: { getOrgInstallation: vi.fn(), @@ -54,6 +63,18 @@ const mockUnmarkOrphan = vi.mocked(mockComputeProvider.unmarkOrphan); const mockMarkIdle = vi.mocked(mockComputeProvider.markIdle); const mockUnmarkIdle = vi.mocked(mockComputeProvider.unmarkIdle); const mockTerminateRunners = vi.mocked(mockComputeProvider.terminate); +const mockGetRunnerStateStore = vi.mocked(getRunnerStateStore); +const mockRunnerStateStore: RunnerStateStore = { + create: vi.fn(), + recordGitHubIdentity: vi.fn(), + activate: vi.fn(), + list: vi.fn(), + markOrphan: vi.fn(), + unmarkOrphan: vi.fn(), + beginTermination: vi.fn(), + cancelTermination: vi.fn(), + delete: vi.fn(), +}; const cleanEnv = process.env; @@ -195,6 +216,14 @@ describe('Scale down runners', () => { mockMarkOrphan.mockResolvedValue(); mockUnmarkOrphan.mockResolvedValue(); mockTerminateRunners.mockResolvedValue(); + mockGetRunnerStateStore.mockReturnValue(undefined); + vi.mocked(mockRunnerStateStore.list).mockResolvedValue([]); + vi.mocked(mockRunnerStateStore.markOrphan).mockResolvedValue(); + vi.mocked(mockRunnerStateStore.unmarkOrphan).mockResolvedValue(); + vi.mocked(mockRunnerStateStore.activate).mockResolvedValue(); + vi.mocked(mockRunnerStateStore.beginTermination).mockResolvedValue('active'); + vi.mocked(mockRunnerStateStore.cancelTermination).mockResolvedValue(); + vi.mocked(mockRunnerStateStore.delete).mockResolvedValue(); mockOctokit.apps.getOrgInstallation.mockImplementation(() => ({ data: { @@ -783,6 +812,309 @@ describe('Scale down runners', () => { expect(mockTerminateRunners).toHaveBeenCalledWith(runners[0].id); }); }); + + describe('with durable runner inventory', () => { + beforeEach(() => { + mockGetRunnerStateStore.mockReturnValue(mockRunnerStateStore); + mockOctokit.actions.getSelfHostedRunnerForOrg.mockResolvedValue({ + data: { id: 101, name: 'runner', busy: false, status: 'online' }, + }); + mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockResolvedValue({ status: 204 }); + }); + + it('reconciles stored-only runners without terminating compute and preserves provider-only bypass removal', async () => { + const providerRunner = createRunnerTestData( + 'provider-only', + 'Org', + MINIMUM_TIME_RUNNING_IN_MINUTES + 1, + true, + false, + false, + ); + providerRunner.bypassRemoval = true; + const storedRecord = createRunnerStateRecord('i-stored-only-org', 'active', { + githubRunnerId: '101', + }); + + mockProviderRunners([providerRunner]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValueOnce([storedRecord]).mockResolvedValueOnce([]); + await scaleDown(); + + expect(mockTerminateRunners).not.toHaveBeenCalledWith(storedRecord.computeResourceId); + expect(mockTerminateRunners).not.toHaveBeenCalledWith(providerRunner.id); + expect(mockOctokit.actions.deleteSelfHostedRunnerFromOrg).toHaveBeenCalledWith({ + org: storedRecord.runnerOwner, + runner_id: 101, + }); + expect(mockRunnerStateStore.delete).toHaveBeenCalledWith(storedRecord.runnerId); + }); + + it('keeps inventory lifecycle canonical when a tracked provider resource has a stale orphan tag', async () => { + const providerRunner = createRunnerTestData('tracked-active', 'Org', MINIMUM_BOOT_TIME - 1, false, true, false); + const storedRecord = createRunnerStateRecord(providerRunner.id, 'active'); + + mockProviderRunners([providerRunner]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValue([storedRecord]); + mockOctokit.paginate.mockResolvedValue([]); + + await scaleDown(); + + expect(mockTerminateRunners).not.toHaveBeenCalled(); + expect(mockUnmarkOrphan).not.toHaveBeenCalled(); + expect(mockRunnerStateStore.beginTermination).not.toHaveBeenCalled(); + }); + + it('claims provider-absent active state before GitHub cleanup and deletes it without compute calls', async () => { + const storedRecord = createRunnerStateRecord('i-active-org', 'active', { githubRunnerId: '101' }); + mockProviderRunners([]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValueOnce([storedRecord]).mockResolvedValueOnce([]); + vi.mocked(mockRunnerStateStore.beginTermination).mockResolvedValueOnce('active'); + await scaleDown(); + + expect(mockRunnerStateStore.beginTermination).toHaveBeenCalledWith(storedRecord.runnerId); + expect(mockTerminateRunners).not.toHaveBeenCalled(); + expect(mockRunnerStateStore.delete).toHaveBeenCalledWith(storedRecord.runnerId); + expect(vi.mocked(mockRunnerStateStore.beginTermination).mock.invocationCallOrder[0]).toBeLessThan( + mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mock.invocationCallOrder[0], + ); + expect(mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(mockRunnerStateStore.delete).mock.invocationCallOrder[0], + ); + }); + + it('treats a GitHub 404 as successful cleanup for provider-absent inventory', async () => { + const storedRecord = createRunnerStateRecord('i-missing-github-runner-org', 'provisioning', { + githubRunnerId: '101', + }); + const error404 = new RequestError('Runner not found', 404, { + request: { + method: 'DELETE', + url: 'https://api.github.com/test', + headers: {}, + }, + }); + mockProviderRunners([]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValueOnce([storedRecord]).mockResolvedValueOnce([]); + vi.mocked(mockRunnerStateStore.beginTermination).mockResolvedValueOnce('provisioning'); + mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockRejectedValueOnce(error404); + + await scaleDown(); + + expect(mockRunnerStateStore.delete).toHaveBeenCalledWith(storedRecord.runnerId); + expect(mockRunnerStateStore.cancelTermination).not.toHaveBeenCalled(); + expect(mockTerminateRunners).not.toHaveBeenCalled(); + expect(mockUnmarkOrphan).not.toHaveBeenCalled(); + expect(mockOctokit.actions.getSelfHostedRunnerForOrg).not.toHaveBeenCalled(); + }); + + it('skips cleanup when another invocation owns the termination claim', async () => { + const storedRecord = createRunnerStateRecord('i-contended-org', 'active', { githubRunnerId: '101' }); + mockProviderRunners([]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValue([storedRecord]); + vi.mocked(mockRunnerStateStore.beginTermination).mockResolvedValueOnce(undefined); + mockOctokit.paginate.mockResolvedValue([{ id: 101, name: storedRecord.computeResourceId }]); + + await scaleDown(); + + expect(mockOctokit.actions.deleteSelfHostedRunnerFromOrg).not.toHaveBeenCalled(); + expect(mockTerminateRunners).not.toHaveBeenCalled(); + expect(mockRunnerStateStore.delete).not.toHaveBeenCalled(); + }); + + it('reclaims a stale terminating provider-absent record without calling compute', async () => { + const terminatingRecord = createRunnerStateRecord('i-stale-terminating-org', 'terminating'); + mockProviderRunners([]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValueOnce([terminatingRecord]).mockResolvedValueOnce([]); + vi.mocked(mockRunnerStateStore.beginTermination).mockResolvedValueOnce('terminating'); + + await scaleDown(); + + expect(mockRunnerStateStore.beginTermination).toHaveBeenCalledWith(terminatingRecord.runnerId); + expect(mockTerminateRunners).not.toHaveBeenCalled(); + expect(mockRunnerStateStore.delete).toHaveBeenCalledWith(terminatingRecord.runnerId); + expect(mockRunnerStateStore.cancelTermination).not.toHaveBeenCalled(); + }); + + it('leaves a reclaimed provider-absent record terminating when state deletion fails', async () => { + const terminatingRecord = createRunnerStateRecord('i-terminating-retry-failure-org', 'terminating'); + mockProviderRunners([]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValue([terminatingRecord]); + vi.mocked(mockRunnerStateStore.beginTermination).mockResolvedValueOnce('terminating'); + vi.mocked(mockRunnerStateStore.delete).mockRejectedValueOnce(new Error('state deletion failed')); + + await scaleDown(); + + expect(mockRunnerStateStore.cancelTermination).not.toHaveBeenCalled(); + expect(mockRunnerStateStore.delete).toHaveBeenCalledWith(terminatingRecord.runnerId); + expect(mockTerminateRunners).not.toHaveBeenCalled(); + }); + + it('restores the exact prior lifecycle state when compute termination fails', async () => { + const providerRunner = createRunnerTestData( + 'stale-provisioning', + 'Org', + MINIMUM_BOOT_TIME + 1, + false, + false, + false, + ); + const staleProvisioning = createRunnerStateRecord(providerRunner.id, 'provisioning'); + mockProviderRunners([providerRunner]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValue([staleProvisioning]); + vi.mocked(mockRunnerStateStore.beginTermination).mockResolvedValueOnce('provisioning'); + mockTerminateRunners.mockRejectedValueOnce(new Error('termination failed')); + + await scaleDown(); + + expect(mockRunnerStateStore.beginTermination).toHaveBeenCalledWith(staleProvisioning.runnerId); + expect(mockRunnerStateStore.cancelTermination).toHaveBeenCalledWith(staleProvisioning.runnerId, 'provisioning'); + expect(mockRunnerStateStore.delete).not.toHaveBeenCalled(); + }); + + it('de-registers a stored orphan before terminating compute and deleting state', async () => { + const providerRunner = createRunnerTestData( + 'orphan-with-github-runner', + 'Org', + MINIMUM_BOOT_TIME + 1, + false, + true, + true, + ); + const storedOrphan = createRunnerStateRecord(providerRunner.id, 'orphan', { + githubRunnerId: '101', + }); + mockProviderRunners([providerRunner]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValueOnce([storedOrphan]).mockResolvedValueOnce([]); + vi.mocked(mockRunnerStateStore.beginTermination).mockResolvedValueOnce('orphan'); + mockOctokit.actions.getSelfHostedRunnerForOrg.mockResolvedValueOnce({ + data: { id: 101, name: storedOrphan.computeResourceId, busy: true, status: 'offline' }, + }); + + await scaleDown(); + + expect(mockOctokit.actions.deleteSelfHostedRunnerFromOrg).toHaveBeenCalledWith({ + org: storedOrphan.runnerOwner, + runner_id: 101, + }); + expect(mockTerminateRunners).toHaveBeenCalledWith(storedOrphan.computeResourceId); + expect(mockRunnerStateStore.delete).toHaveBeenCalledWith(storedOrphan.runnerId); + expect(mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mock.invocationCallOrder[0]).toBeLessThan( + mockTerminateRunners.mock.invocationCallOrder[0], + ); + expect(mockTerminateRunners.mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(mockRunnerStateStore.delete).mock.invocationCallOrder[0], + ); + }); + + it('restores an orphan claim when GitHub cleanup fails', async () => { + const storedOrphan = createRunnerStateRecord('i-orphan-cleanup-failure-org', 'orphan', { + githubRunnerId: '101', + }); + mockProviderRunners([]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValue([storedOrphan]); + vi.mocked(mockRunnerStateStore.beginTermination).mockResolvedValueOnce('orphan'); + mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockResolvedValueOnce({ status: 500 }); + + await scaleDown(); + + expect(mockRunnerStateStore.cancelTermination).toHaveBeenCalledWith(storedOrphan.runnerId, 'orphan'); + expect(mockTerminateRunners).not.toHaveBeenCalled(); + expect(mockRunnerStateStore.delete).not.toHaveBeenCalled(); + }); + + it('does not restore state when deletion fails after compute termination succeeds', async () => { + const providerRunner = createRunnerTestData( + 'delete-failure', + 'Org', + MINIMUM_TIME_RUNNING_IN_MINUTES + 1, + true, + false, + true, + ); + const storedRecord = createRunnerStateRecord(providerRunner.id, 'active', { githubRunnerId: '101' }); + mockProviderRunners([providerRunner]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValue([storedRecord]); + vi.mocked(mockRunnerStateStore.beginTermination).mockResolvedValueOnce('active'); + vi.mocked(mockRunnerStateStore.delete).mockRejectedValueOnce(new Error('delete failed')); + mockOctokit.paginate.mockResolvedValue([{ id: 101, name: storedRecord.computeResourceId }]); + + await expect(scaleDown()).resolves.not.toThrow(); + + expect(mockTerminateRunners).toHaveBeenCalledWith(providerRunner.id); + expect(mockRunnerStateStore.cancelTermination).not.toHaveBeenCalled(); + }); + + it('updates durable lifecycle only after provider orphan removal succeeds', async () => { + const providerRunner = createRunnerTestData( + 'tracked-orphan', + 'Org', + MINIMUM_BOOT_TIME + 1, + false, + true, + false, + undefined, + 101, + ); + const storedOrphan = createRunnerStateRecord(providerRunner.id, 'orphan', { githubRunnerId: '101' }); + mockProviderRunners([providerRunner]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValue([storedOrphan]); + mockOctokit.actions.getSelfHostedRunnerForOrg.mockResolvedValueOnce({ + data: { id: 101, name: providerRunner.id, busy: false, status: 'online' }, + }); + + await scaleDown(); + + expect(mockUnmarkOrphan).toHaveBeenCalledWith(providerRunner.id); + expect(mockRunnerStateStore.unmarkOrphan).toHaveBeenCalledWith(storedOrphan.runnerId); + expect(mockUnmarkOrphan.mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(mockRunnerStateStore.unmarkOrphan).mock.invocationCallOrder[0], + ); + expect(mockTerminateRunners).not.toHaveBeenCalled(); + }); + + it('retains durable orphan state when provider orphan removal fails', async () => { + const providerRunner = createRunnerTestData( + 'tracked-orphan-failure', + 'Org', + MINIMUM_BOOT_TIME + 1, + false, + true, + false, + undefined, + 101, + ); + const storedOrphan = createRunnerStateRecord(providerRunner.id, 'orphan', { githubRunnerId: '101' }); + mockProviderRunners([providerRunner]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValue([storedOrphan]); + mockOctokit.actions.getSelfHostedRunnerForOrg.mockResolvedValueOnce({ + data: { id: 101, name: providerRunner.id, busy: false, status: 'online' }, + }); + mockUnmarkOrphan.mockRejectedValueOnce(new Error('provider unmark failed')); + + await scaleDown(); + + expect(mockRunnerStateStore.unmarkOrphan).not.toHaveBeenCalled(); + expect(mockRunnerStateStore.activate).not.toHaveBeenCalled(); + expect(mockTerminateRunners).not.toHaveBeenCalled(); + }); + + it('reconciles expired provisioning records but leaves fresh provisioning records alone', async () => { + const staleProvisioning = createRunnerStateRecord('i-stale-provisioning-org', 'provisioning'); + const freshProvisioning = createRunnerStateRecord('i-fresh-provisioning-org', 'provisioning', { + createdAt: new Date().toISOString(), + }); + mockProviderRunners([]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValue([staleProvisioning, freshProvisioning]); + vi.mocked(mockRunnerStateStore.beginTermination).mockResolvedValueOnce('provisioning'); + + await scaleDown(); + + expect(mockRunnerStateStore.beginTermination).toHaveBeenCalledTimes(1); + expect(mockRunnerStateStore.beginTermination).toHaveBeenCalledWith(staleProvisioning.runnerId); + expect(mockTerminateRunners).not.toHaveBeenCalled(); + expect(mockRunnerStateStore.delete).toHaveBeenCalledWith(staleProvisioning.runnerId); + }); + }); }); function mockProviderRunners(runners: RunnerTestItem[]) { @@ -843,3 +1175,24 @@ function createRunnerTestData( bypassRemoval: false, }; } + +function createRunnerStateRecord( + runnerId: string, + state: RunnerStateRecord['state'], + overrides: Partial = {}, +): RunnerStateRecord { + const timestamp = moment(new Date()) + .subtract(MINIMUM_TIME_RUNNING_IN_MINUTES + 1, 'minutes') + .toISOString(); + return { + runnerId, + computeProvider: 'ec2', + computeResourceId: runnerId, + runnerOwner: TEST_DATA.repositoryOwner, + runnerType: 'Org', + state, + createdAt: timestamp, + updatedAt: timestamp, + ...overrides, + }; +} diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts index 1e3e838aed..aff736bed1 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts @@ -3,6 +3,11 @@ import { Endpoints } from '@octokit/types'; import { RequestError } from '@octokit/request-error'; import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; import { resolveComputeProviderType } from '@aws-github-runner/compute-providers/provider-types'; +import { + getRunnerStateStore, + type RunnerStateRecord, + type RunnerStateStore, +} from '@aws-github-runner/storage-providers'; import moment from 'moment'; import { @@ -22,7 +27,19 @@ const logger = createChildLogger('scale-down'); type OrgRunnerList = Endpoints['GET /orgs/{org}/actions/runners']['response']['data']['runners']; type RepoRunnerList = Endpoints['GET /repos/{owner}/{repo}/actions/runners']['response']['data']['runners']; -type RunnerState = OrgRunnerList[number] | RepoRunnerList[number]; +type GitHubRunnerState = OrgRunnerList[number] | RepoRunnerList[number]; + +interface InventoryRunnerInfo extends RunnerInfo { + inventory?: RunnerStateRecord; + providerPresent?: boolean; +} + +type RestorableRunnerState = Parameters[1]; + +interface TerminationClaim { + runnerStateId?: string; + restoreState?: RestorableRunnerState; +} async function getOrCreateOctokit(runner: RunnerInfo): Promise { const key = runner.owner; @@ -67,7 +84,7 @@ async function getGitHubSelfHostedRunnerState( client: Octokit, runner: RunnerInfo, runnerId: number, -): Promise { +): Promise { try { const state = runner.type === 'Org' @@ -161,6 +178,12 @@ async function deleteGitHubRunner( } return { ghRunnerId, status: response.status, success: response.status === 204 }; } catch (error) { + if (error instanceof RequestError && error.status === 404) { + logger.info( + `GitHub runner ${ghRunnerId} for runner '${runner.id}' is already de-registered; treating cleanup as complete.`, + ); + return { ghRunnerId, status: error.status, success: true }; + } logger.error( `Failed to de-register GitHub runner ${ghRunnerId} for runner '${runner.id}'. ` + `Error: ${error instanceof Error ? error.message : String(error)}`, @@ -223,11 +246,12 @@ async function clearIdleDetection(runner: RunnerInfo, computeProvider: ScaleDown } async function removeRunner( - runner: RunnerInfo, + runner: InventoryRunnerInfo, ghRunnerIds: number[], computeProvider: ScaleDownComputeProvider, ): Promise { - const githubInstallationClient = await getOrCreateOctokit(runner); + let terminationClaim: TerminationClaim | undefined; + let computeTerminated = false; try { if (runner.bypassRemoval) { logger.info( @@ -236,6 +260,7 @@ async function removeRunner( return; } + const githubInstallationClient = await getOrCreateOctokit(runner); const states = await Promise.all( ghRunnerIds.map(async (ghRunnerId) => { // Get busy state instead of using the output of listGitHubRunners(...) to minimize to race condition. @@ -247,6 +272,14 @@ async function removeRunner( if (!(await idleConfirmed(runner, computeProvider))) { return; } + + const claim = await beginRunnerTermination(runner); + if (claim === undefined) { + logger.info(`Runner '${runner.id}' is already being reconciled; skipping this scale-down cycle.`); + return; + } + terminationClaim = claim; + const results = await Promise.all( ghRunnerIds.map((ghRunnerId) => deleteGitHubRunner(githubInstallationClient, runner, ghRunnerId)), ); @@ -256,10 +289,15 @@ async function removeRunner( if (allSucceeded) { await computeProvider.terminate(runner.id); + computeTerminated = true; + await completeRunnerTermination(terminationClaim); + terminationClaim = undefined; logger.info( `${computeProvider.type.toUpperCase()} runner '${runner.id}' is terminated and GitHub runner is de-registered.`, ); } else { + await restoreRunnerTermination(terminationClaim); + terminationClaim = undefined; // Only terminate the provider runner if it was successfully de-registered from GitHub. logger.error( `Failed to de-register ${failedRunners.length} GitHub runner(s) for runner '${runner.id}'. ` + @@ -272,6 +310,9 @@ async function removeRunner( logger.info(`Runner '${runner.id}' cannot be de-registered, because it is still busy.`); } } catch (e) { + if (terminationClaim && !computeTerminated) { + await restoreRunnerTermination(terminationClaim); + } logger.error( `Runner '${runner.id}' cannot be de-registered. Error: ${e instanceof Error ? e.message : String(e)}`, { error: e }, @@ -280,7 +321,7 @@ async function removeRunner( } async function evaluateAndRemoveRunners( - runners: RunnerInfo[], + runners: InventoryRunnerInfo[], scaleDownConfigs: ScalingDownConfigList, computeProvider: ScaleDownComputeProvider, ): Promise { @@ -293,14 +334,27 @@ async function evaluateAndRemoveRunners( .filter((runner) => runner.owner === ownerTag) .sort(evictionStrategy === 'oldest_first' ? oldestFirstStrategy : newestFirstStrategy); logger.debug(`Found: '${ownerRunners.length}' active GitHub runners with owner tag: '${ownerTag}'`); - logger.debug(`Active GitHub runners with owner tag: '${ownerTag}': ${JSON.stringify(ownerRunners)}`); + if (ownerRunners.some((runner) => runner.inventory)) { + logger.debug(`Active GitHub runner inventory with owner tag: '${ownerTag}'`, { + runners: ownerRunners.map((runner) => ({ + computeResourceId: runner.id, + lifecycleState: runner.inventory?.state, + })), + }); + } else { + logger.debug(`Active GitHub runners with owner tag: '${ownerTag}': ${JSON.stringify(ownerRunners)}`); + } for (const runner of ownerRunners) { if (runner.bypassRemoval) { logger.debug(`Runner '${runner.id}' has bypass-removal tag set, skipping evaluation.`); continue; } const ghRunners = await listGitHubRunners(runner); - const ghRunnersFiltered = ghRunners.filter((ghRunner: { name: string }) => ghRunner.name.endsWith(runner.id)); + const ghRunnersFiltered = ghRunners.filter((ghRunner: { id: number; name: string }) => + runner.inventory?.githubRunnerId + ? ghRunner.id.toString() === runner.inventory.githubRunnerId + : ghRunner.name.endsWith(runner.id), + ); logger.debug(`Found: '${ghRunnersFiltered.length}' GitHub runners for runner: '${runner.id}'`); logger.debug(`GitHub runners for runner: '${runner.id}': ${JSON.stringify(ghRunnersFiltered)}`); if (ghRunnersFiltered.length) { @@ -321,7 +375,7 @@ async function evaluateAndRemoveRunners( } } } else if (computeProvider.bootTimeExceeded(runner)) { - await markOrphan(runner.id, computeProvider); + await markOrphan(runner, computeProvider); } else { logger.debug(`Runner ${runner.id} has not yet booted.`); } @@ -329,25 +383,46 @@ async function evaluateAndRemoveRunners( } } -async function markOrphan(id: string, computeProvider: ScaleDownComputeProvider): Promise { +async function markOrphan(runner: InventoryRunnerInfo, computeProvider: ScaleDownComputeProvider): Promise { try { - await computeProvider.markOrphan(id); - logger.info(`Runner '${id}' tagged as orphan.`); + if (runner.inventory) { + await getRunnerStateStore()?.markOrphan(runner.inventory.runnerId); + } + await computeProvider.markOrphan(runner.id); + logger.info(`Runner '${runner.id}' tagged as orphan.`); } catch (e) { - logger.error(`Failed to tag runner '${id}' as orphan.`, { error: e }); + logger.error(`Failed to tag runner '${runner.id}' as orphan.`, { error: e }); } } -async function unMarkOrphan(id: string, computeProvider: ScaleDownComputeProvider): Promise { +async function unMarkOrphan(runner: InventoryRunnerInfo, computeProvider: ScaleDownComputeProvider): Promise { try { - await computeProvider.unmarkOrphan(id); - logger.info(`Runner '${id}' untagged as orphan.`); + // Keep the durable lifecycle unchanged if the provider mutation fails. In + // particular, activating a provisioning record removes its safety TTL. + await computeProvider.unmarkOrphan(runner.id); + const runnerStateStore = getRunnerStateStore(); + if (runnerStateStore && runner.inventory?.state === 'orphan') { + await runnerStateStore.unmarkOrphan(runner.inventory.runnerId); + } else if (runnerStateStore && runner.inventory?.state === 'provisioning') { + await runnerStateStore.activate(runner.inventory.runnerId, { + githubRunnerId: runner.githubRunnerId, + runnerLabels: runner.inventory.runnerLabels, + runnerName: runner.inventory.runnerName, + metadata: runner.inventory.metadata, + }); + } + logger.info(`Runner '${runner.id}' untagged as orphan.`); } catch (e) { - logger.error(`Failed to un-tag runner '${id}' as orphan.`, { error: e }); + logger.error(`Failed to un-tag runner '${runner.id}' as orphan.`, { error: e }); } } -async function lastChanceCheckOrphanRunner(runner: RunnerInfo): Promise { +interface OrphanEvaluation { + isOrphan: boolean; + githubRunnerExists: boolean; +} + +async function lastChanceCheckOrphanRunner(runner: InventoryRunnerInfo): Promise { const client = await getOrCreateOctokit(runner); const runnerId = parseInt(runner.githubRunnerId || '0'); const state = await getGitHubSelfHostedRunnerState(client, runner, runnerId); @@ -364,30 +439,52 @@ async function lastChanceCheckOrphanRunner(runner: RunnerInfo): Promise } } logger.info(`Runner '${runner.id}' is judged to ${isOrphan ? 'be' : 'not be'} orphaned.`); - return isOrphan; + return { isOrphan, githubRunnerExists: state !== null }; } -async function terminateOrphan(environment: string, computeProvider: ScaleDownComputeProvider): Promise { +async function terminateOrphan( + environment: string, + computeProvider: ScaleDownComputeProvider, + runners?: InventoryRunnerInfo[], +): Promise { try { - const orphanRunners = await computeProvider.list(environment, true); + const orphanRunners: InventoryRunnerInfo[] = runners ?? (await computeProvider.list(environment, true)); for (const runner of orphanRunners) { if (runner.bypassRemoval) { logger.info(`Orphan runner '${runner.id}' has bypass-removal tag set, skipping termination.`); continue; } + if (runner.inventory?.state === 'provisioning' && !computeProvider.bootTimeExceeded(runner)) { + logger.debug(`Runner '${runner.id}' is still provisioning; skipping reconciliation until boot time expires.`); + continue; + } if (runner.githubRunnerId) { - const isOrphan = await lastChanceCheckOrphanRunner(runner); - if (isOrphan) { - await computeProvider.terminate(runner.id); + const orphanEvaluation = await lastChanceCheckOrphanRunner(runner); + if (orphanEvaluation.isOrphan) { + if (runner.inventory) { + await terminateClaimedRunner( + runner, + computeProvider, + orphanEvaluation.githubRunnerExists ? parseInt(runner.githubRunnerId) : undefined, + ); + } else { + // Preserve the provider-only recovery behavior used by the legacy path. + await computeProvider.terminate(runner.id); + } } else { - await unMarkOrphan(runner.id, computeProvider); + await unMarkOrphan(runner, computeProvider); } } else { logger.info(`Terminating orphan runner '${runner.id}'`); - await computeProvider.terminate(runner.id).catch((e) => { - logger.error(`Failed to terminate orphan runner '${runner.id}'`, { error: e }); - }); + if (runner.inventory) { + await terminateClaimedRunner(runner, computeProvider); + } else { + // Preserve the provider-only recovery behavior used by the legacy path. + await computeProvider.terminate(runner.id).catch((error) => { + logger.error(`Failed to terminate orphan runner '${runner.id}'`, { error }); + }); + } } } } catch (e) { @@ -395,6 +492,136 @@ async function terminateOrphan(environment: string, computeProvider: ScaleDownCo } } +async function terminateClaimedRunner( + runner: InventoryRunnerInfo, + computeProvider: ScaleDownComputeProvider, + githubRunnerId?: number, +): Promise { + const claim = await beginRunnerTermination(runner); + if (claim === undefined) { + logger.info(`Runner '${runner.id}' is already being reconciled; skipping this scale-down cycle.`); + return; + } + + try { + if (githubRunnerId !== undefined) { + const githubInstallationClient = await getOrCreateOctokit(runner); + const result = await deleteGitHubRunner(githubInstallationClient, runner, githubRunnerId); + if (!result.success) { + await restoreRunnerTermination(claim); + logger.error( + `Failed to de-register GitHub runner '${githubRunnerId}' for orphan runner '${runner.id}'. ` + + `Runner will NOT be terminated to allow retry on next scale-down cycle.`, + ); + return; + } + } + await computeProvider.terminate(runner.id); + } catch (error) { + await restoreRunnerTermination(claim); + logger.error(`Failed to clean up orphan runner '${runner.id}'`, { error }); + return; + } + + try { + await completeRunnerTermination(claim); + } catch (error) { + logger.error(`Failed to delete runner state for terminated runner '${runner.id}'`, { error }); + } +} + +async function reconcileProviderAbsentRunner( + runner: InventoryRunnerInfo, + computeProvider: ScaleDownComputeProvider, +): Promise { + if (!runner.inventory || runner.providerPresent !== false) { + return; + } + + // Provider discovery can lag immediately after launch. Retain the inventory + // until the normal boot grace has elapsed before declaring compute absent. + if (!computeProvider.bootTimeExceeded(runner)) { + logger.debug(`Runner '${runner.id}' is absent from provider discovery but remains inside its boot grace.`); + return; + } + + const claim = await beginRunnerTermination(runner); + if (claim === undefined) { + logger.info(`Runner '${runner.id}' is already being reconciled; skipping this scale-down cycle.`); + return; + } + + try { + if (runner.githubRunnerId) { + const githubInstallationClient = await getOrCreateOctokit(runner); + const result = await deleteGitHubRunner(githubInstallationClient, runner, parseInt(runner.githubRunnerId, 10)); + if (!result.success) { + await restoreRunnerTermination(claim); + logger.error( + `Failed to de-register GitHub runner '${runner.githubRunnerId}' for provider-absent runner ` + + `'${runner.id}'. Runner state will be retained for retry.`, + ); + return; + } + } + } catch (error) { + await restoreRunnerTermination(claim); + logger.error(`Failed to reconcile GitHub state for provider-absent runner '${runner.id}'`, { error }); + return; + } + + // Provider discovery already established that the compute resource is gone; + // avoid tag/terminate calls that would turn idempotent cleanup into a retry loop. + try { + await completeRunnerTermination(claim); + logger.info(`Removed durable state for provider-absent runner '${runner.id}'.`); + } catch (error) { + // GitHub cleanup is committed at this point. Leave the terminating claim in + // place so its lease/TTL makes state deletion retryable without resurrection. + logger.error(`Failed to delete runner state for provider-absent runner '${runner.id}'`, { error }); + } +} + +async function beginRunnerTermination(runner: InventoryRunnerInfo): Promise { + if (!runner.inventory) { + return {}; + } + + const runnerStateStore = getRunnerStateStore(); + if (!runnerStateStore) { + return {}; + } + + const previousState = await runnerStateStore.beginTermination(runner.inventory.runnerId); + if (previousState === undefined) { + return undefined; + } + return { + runnerStateId: runner.inventory.runnerId, + // A reclaimed terminating record has no known pre-claim lifecycle state. + // Leave it terminating on failure so the lease can make it retryable again. + ...(previousState === 'terminating' ? {} : { restoreState: previousState }), + }; +} + +async function restoreRunnerTermination(claim: TerminationClaim): Promise { + if (!claim.runnerStateId || !claim.restoreState) { + return; + } + + try { + await getRunnerStateStore()?.cancelTermination(claim.runnerStateId, claim.restoreState); + } catch (error) { + logger.error(`Failed to restore runner state for '${claim.runnerStateId}' after termination failure.`, { error }); + } +} + +async function completeRunnerTermination(claim: TerminationClaim): Promise { + if (claim.runnerStateId) { + await getRunnerStateStore()?.delete(claim.runnerStateId); + } +} + export function oldestFirstStrategy(a: RunnerInfo, b: RunnerInfo): number { if (a.launchTime === undefined) return 1; if (b.launchTime === undefined) return 1; @@ -411,12 +638,52 @@ async function listRunners(environment: string, computeProvider: ScaleDownComput return await computeProvider.list(environment); } -function filterRunners(runners: RunnerInfo[]): RunnerInfo[] { +function filterRunners(runners: InventoryRunnerInfo[]): InventoryRunnerInfo[] { // Managed runners are launched with owner and type tags together. Exclude incomplete records because both // values are required to select the GitHub owner and runner API used during scale-down. return runners.filter((runner) => runner.owner && runner.type && !runner.orphan); } +function mergeRunnerInventory(records: RunnerStateRecord[], providerRunners: RunnerInfo[]): InventoryRunnerInfo[] { + const runnersByComputeResource = new Map(); + for (const record of records) { + runnersByComputeResource.set(record.computeResourceId, { + id: record.computeResourceId, + launchTime: new Date(record.createdAt), + owner: record.runnerOwner, + type: record.runnerType, + orphan: record.state !== 'active', + githubRunnerId: record.githubRunnerId, + inventory: record, + providerPresent: false, + }); + } + + for (const providerRunner of providerRunners) { + const storedRunner = runnersByComputeResource.get(providerRunner.id); + if (!storedRunner) { + runnersByComputeResource.set(providerRunner.id, { ...providerRunner, providerPresent: true }); + continue; + } + + runnersByComputeResource.set(providerRunner.id, { + ...storedRunner, + ...providerRunner, + id: storedRunner.id, + owner: storedRunner.owner, + type: storedRunner.type, + // Lifecycle state is canonical for tracked resources. Provider tags are + // retained as recovery data only for resources missing from inventory. + orphan: storedRunner.orphan, + githubRunnerId: storedRunner.githubRunnerId ?? providerRunner.githubRunnerId, + inventory: storedRunner.inventory, + providerPresent: true, + }); + } + + return Array.from(runnersByComputeResource.values()); +} + export async function scaleDown(): Promise { githubCache.reset(); const environment = process.env.ENVIRONMENT; @@ -426,28 +693,70 @@ export async function scaleDown(): Promise { ...controlPlaneProviderRegistry.capability(computeProviderType, 'scaleDown')(), type: computeProviderType, }; + const runnerStateStore = getRunnerStateStore(); + let managedRunners: InventoryRunnerInfo[]; + let runnersToEvaluate: InventoryRunnerInfo[]; + + if (!runnerStateStore) { + // Preserve the legacy SSM-backed lifecycle. EC2 remains the source of truth + // when no durable runner inventory is configured. + await terminateOrphan(environment, computeProvider); + managedRunners = await listRunners(environment, computeProvider); + runnersToEvaluate = managedRunners; + } else { + const providerRunners = await listRunners(environment, computeProvider); + const inventoryRecords = await runnerStateStore.list({ computeProvider: computeProvider.type }); + managedRunners = mergeRunnerInventory(inventoryRecords, providerRunners); - // first runners marked to be orphan. - await terminateOrphan(environment, computeProvider); + const providerAbsentRunners = managedRunners.filter( + (runner) => runner.inventory !== undefined && runner.providerPresent === false, + ); + for (const runner of providerAbsentRunners) { + await reconcileProviderAbsentRunner(runner, computeProvider); + } + runnersToEvaluate = managedRunners.filter( + (runner) => runner.inventory === undefined || runner.providerPresent !== false, + ); + + // Reconcile stale provisioning and orphan records first. Provider tags remain + // a recovery signal for resources that predate the durable inventory. + await terminateOrphan( + environment, + computeProvider, + runnersToEvaluate.filter((runner) => runner.orphan), + ); + } - // next scale down idle runners with respect to config and mark potential orphans - const providerRunners = await listRunners(environment, computeProvider); - const activeProviderRunnersCount = providerRunners.length; + const runnerCountLabel = runnerStateStore ? 'managed' : 'active'; + const managedRunnerCount = managedRunners.length; logger.info( - `Found: '${activeProviderRunnersCount}' active ${computeProvider.type.toUpperCase()} runners before clean-up.`, + `Found: '${managedRunnerCount}' ${runnerCountLabel} ${computeProvider.type.toUpperCase()} runners before clean-up.`, ); - logger.debug(`Active ${computeProvider.type.toUpperCase()} runners: ${JSON.stringify(providerRunners)}`); + if (runnerStateStore) { + logger.debug(`Active ${computeProvider.type.toUpperCase()} runner inventory`, { + runners: managedRunners.map((runner) => ({ + computeResourceId: runner.id, + lifecycleState: runner.inventory?.state, + })), + }); + } else { + logger.debug(`Active ${computeProvider.type.toUpperCase()} runners: ${JSON.stringify(managedRunners)}`); + } - if (activeProviderRunnersCount === 0) { - logger.debug(`No active runners found for environment: '${environment}'`); + if (managedRunnerCount === 0) { + logger.debug(`No ${runnerCountLabel} runners found for environment: '${environment}'`); return; } - const runners = filterRunners(providerRunners); + const runners = filterRunners(runnersToEvaluate); await evaluateAndRemoveRunners(runners, scaleDownConfigs, computeProvider); - const activeProviderRunnersCountAfter = (await listRunners(environment, computeProvider)).length; + const providerRunnersAfter = await listRunners(environment, computeProvider); + const managedRunnerCountAfter = runnerStateStore + ? mergeRunnerInventory(await runnerStateStore.list({ computeProvider: computeProvider.type }), providerRunnersAfter) + .length + : providerRunnersAfter.length; logger.info( - `Found: '${activeProviderRunnersCountAfter}' active ${computeProvider.type.toUpperCase()} runners after clean-up.`, + `Found: '${managedRunnerCountAfter}' ${runnerCountLabel} ${computeProvider.type.toUpperCase()} runners after clean-up.`, ); } diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up-contract.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up-contract.test.ts index d7a025fd88..055ba0db5c 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up-contract.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up-contract.test.ts @@ -15,6 +15,15 @@ vi.mock('../github/auth', () => ({ createOctokitClient: vi.fn(), })); +vi.mock('@aws-github-runner/storage-providers', () => ({ + createStorageProviders: vi.fn().mockReturnValue({ + githubAppCredentials: { get: vi.fn() }, + }), + getRunnerConfigStore: vi.fn(), + getRunnerGroupCacheStore: vi.fn(), + getRunnerStateStore: vi.fn().mockReturnValue(undefined), +})); + vi.mock('./github-runner', async (importOriginal) => ({ ...(await importOriginal()), getGitHubEnterpriseApiUrl: vi.fn(), diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts index 664bac60fb..f3c93a4227 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts @@ -52,7 +52,7 @@ interface TestRunnerLookupInput { } const createRunner = vi.fn<(input: TestRunnerCreationInput) => Promise>(); -const listRunners = vi.fn<(input: TestRunnerLookupInput) => Promise>(); +const listRunners = vi.fn<(input: TestRunnerLookupInput) => Promise<{ id: string }[]>>(); const mockCreateRunner = vi.mocked(createRunner); const mockListRunners = vi.mocked(listRunners); const mockSSMClient = mockClient(SSMClient); @@ -207,7 +207,7 @@ beforeEach(() => { runnerType: input.runnerType, runnerOwner: input.runnerOwner, }) - ).length; + ).map(({ id }) => id); }); mockCreateRunners.mockImplementation(createTestProviderRunners); diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts index d4e3889f19..719bed0de6 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts @@ -1,7 +1,11 @@ import { addPersistentContextToChildLogger, createChildLogger } from '@aws-github-runner/aws-powertools-util'; import { InvalidRunnerLabelsError } from '@aws-github-runner/compute-providers/core'; import { resolveComputeProviderType } from '@aws-github-runner/compute-providers/provider-types'; -import { createStorageProviders, type StorageProviders } from '@aws-github-runner/storage-providers'; +import { + createStorageProviders, + getRunnerStateStore, + type StorageProviders, +} from '@aws-github-runner/storage-providers'; import { Octokit } from '@octokit/rest'; import yn from 'yn'; @@ -289,7 +293,13 @@ export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise + await computeProvider.getCurrentRunners(runnerLabelResolution.state, { runnerType, runnerOwner }), + ); logger.info('Current runners', { currentRunners, @@ -400,6 +410,31 @@ export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise Promise, +): Promise { + const runnerStateStore = getRunnerStateStore(); + if (!runnerStateStore) { + return (await getProviderRunnerIds()).length; + } + + const records = await runnerStateStore.list({ computeProvider }); + const runnerIds = new Set( + records + .filter((record) => record.runnerType === runnerType && record.runnerOwner === runnerOwner) + .map((record) => record.computeResourceId), + ); + // Durable inventory is canonical. Provider discovery remains a conservative + // recovery source for resources launched before their inventory record was written. + for (const runnerId of await getProviderRunnerIds()) { + runnerIds.add(runnerId); + } + return runnerIds.size; +} + function isValidRepoOwnerTypeIfOrgLevelEnabled(payload: ActionRequestMessage, enableOrgLevel: boolean): boolean { return !(enableOrgLevel && payload.repoOwnerType !== 'Organization'); } diff --git a/lambdas/functions/control-plane/src/test/compute-provider-contracts/scale-up.ts b/lambdas/functions/control-plane/src/test/compute-provider-contracts/scale-up.ts index ecab384118..542f01d726 100644 --- a/lambdas/functions/control-plane/src/test/compute-provider-contracts/scale-up.ts +++ b/lambdas/functions/control-plane/src/test/compute-provider-contracts/scale-up.ts @@ -43,7 +43,7 @@ export function defineScaleUpContractTests({ resolveCapability.mockReturnValue(() => provider); vi.mocked(provider.resolveLabelsForRunners).mockResolvedValue({ runnerLabels: [], state }); - vi.mocked(provider.getCurrentRunners).mockResolvedValue(0); + vi.mocked(provider.getCurrentRunners).mockResolvedValue([]); vi.mocked(provider.createRunners).mockResolvedValue(createResult); }); @@ -81,7 +81,7 @@ export function defineScaleUpContractTests({ it('does not create runners when the compute provider has reached maximum capacity', async () => { process.env.RUNNERS_MAXIMUM_COUNT = '1'; - vi.mocked(provider.getCurrentRunners).mockResolvedValue(1); + vi.mocked(provider.getCurrentRunners).mockResolvedValue(['runner-1']); await scaleUp(createPayloads()); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts index 124b676ae6..7b1e41cbbc 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts @@ -155,6 +155,9 @@ function createEc2StartRunnerConfigOptions( return { runnerConfigStore: storage?.runnerConfig, runnerGroupCacheStore: storage?.runnerGroupCache, + computeProvider: 'ec2', + getRunnerConfigAccessScope: (instanceId) => + process.env.EC2_INSTANCE_ARN_PREFIX ? `${process.env.EC2_INSTANCE_ARN_PREFIX}${instanceId}` : undefined, getRunnerConfigMetadata: (instanceId) => [{ key: 'InstanceId', value: instanceId }], onJitConfigCreated: async (instanceId, metadata) => await tagEc2RunnerMetadata(ec2Operations, instanceId, metadata), }; diff --git a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts index 2a79eea40d..663e2ed5b3 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts @@ -106,7 +106,7 @@ async function expectCurrentRunners(runnerType: RunnerType, owner: string) { runnerType, runnerOwner: owner, }), - ).resolves.toBe(1); + ).resolves.toEqual(['i-1234']); expect(mockListRunners).toHaveBeenCalledWith({ environment: 'unit-test-environment', runnerType, diff --git a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.ts index a27758b574..ebc2a6a263 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.ts @@ -59,7 +59,9 @@ export function createEc2ScaleUpCapability( return { resolveLabelsForRunners: (labels) => resolveEc2ScaleUpRunnerLabels(ec2Operations, labels), getCurrentRunners: async (_state, { runnerType, runnerOwner }) => - (await ec2Operations.list({ environment: process.env.ENVIRONMENT, runnerType, runnerOwner })).length, + (await ec2Operations.list({ environment: process.env.ENVIRONMENT, runnerType, runnerOwner })).map( + (runner) => runner.id, + ), createRunners: async ({ githubRunnerConfig, numberOfRunners, githubInstallationClient, state, storage }) => { const config = loadEc2ScaleUpProviderConfig(); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/environment.d.ts b/lambdas/libs/compute-providers/aws/ec2/src/environment.d.ts index 71ee01ff2f..189b161925 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/environment.d.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/environment.d.ts @@ -16,6 +16,7 @@ declare global { | 'capacity-optimized' | 'capacity-optimized-prioritized' | 'prioritized'; + EC2_INSTANCE_ARN_PREFIX: string | undefined; SCALE_ERRORS: string; } } diff --git a/lambdas/libs/compute-providers/core/index.ts b/lambdas/libs/compute-providers/core/index.ts index 2457719a42..365a0dfcf5 100644 --- a/lambdas/libs/compute-providers/core/index.ts +++ b/lambdas/libs/compute-providers/core/index.ts @@ -31,6 +31,10 @@ export interface GitHubRunnerMetadata { export interface StartRunnerConfigOptions { runnerConfigStore?: import('@aws-github-runner/storage-providers').RunnerConfigStore; runnerGroupCacheStore?: import('@aws-github-runner/storage-providers').RunnerGroupCacheStore; + /** Compute provider that owns the runner IDs. Used by provider-neutral runner inventory. */ + computeProvider?: ComputeProviderType; + /** Access scope used by secret stores to isolate one runner's bootstrap configuration. */ + getRunnerConfigAccessScope?: (runnerId: string) => string | undefined; getRunnerConfigMetadata?: (runnerId: string) => { key: string; value: string }[]; onJitConfigCreated?: (runnerId: string, metadata: GitHubRunnerMetadata) => Promise; } @@ -76,7 +80,8 @@ export interface CreateRunnerResult { export interface ScaleUpComputeProvider extends ComputeProvider { resolveLabelsForRunners(messageLabels: string[]): Promise>; - getCurrentRunners(state: TState, input: CurrentRunnersInput): Promise; + /** Compute resource IDs currently owned by this runner entry. */ + getCurrentRunners(state: TState, input: CurrentRunnersInput): Promise; createRunners(input: CreateScaleUpRunnersInput): Promise; } @@ -138,7 +143,7 @@ export interface CreatePoolRunnersInput { storage?: import('@aws-github-runner/storage-providers').RunnerConfigStorage; } -export interface PoolComputeProvider extends ComputeProvider { +export interface PoolComputeProvider extends ComputeProvider { listRunners(input: ListPoolRunnersInput): Promise; countAvailableRunners( runners: TRunner[], diff --git a/lambdas/libs/storage-providers/aws/dynamodb/client.ts b/lambdas/libs/storage-providers/aws/dynamodb/client.ts new file mode 100644 index 0000000000..86493cc23f --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/client.ts @@ -0,0 +1,21 @@ +import { getTracedAWSV3Client } from '@aws-github-runner/aws-powertools-util'; +import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; + +let memoisedClient: DynamoDBClient | undefined; + +export function getDynamoDbClient(): DynamoDBClient { + memoisedClient ??= getTracedAWSV3Client( + new DynamoDBClient({ + region: process.env.AWS_REGION, + maxAttempts: 10, + // One client serves two tables, so avoid an adaptive rate bucket coupling their throttling behavior. + retryMode: 'standard', + }), + ); + return memoisedClient; +} + +// Test-only reset for cases that need a fresh AWS SDK client. +export function resetDynamoDbClient(): void { + memoisedClient = undefined; +} diff --git a/lambdas/libs/storage-providers/aws/dynamodb/durable-config.ts b/lambdas/libs/storage-providers/aws/dynamodb/durable-config.ts new file mode 100644 index 0000000000..f22fde3269 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/durable-config.ts @@ -0,0 +1,37 @@ +import { GetItemCommand } from '@aws-sdk/client-dynamodb'; + +import { getDynamoDbClient } from './client'; +import { ID_ATTRIBUTE, SCOPE_ATTRIBUTE, VALUE_ATTRIBUTE } from './keys'; + +export async function getDurableConfigValue( + tableName: string, + scope: string, + id: string, + description: string, +): Promise { + const result = await getDynamoDbClient().send( + new GetItemCommand({ + TableName: tableName, + Key: { + [SCOPE_ATTRIBUTE]: { S: scope }, + [ID_ATTRIBUTE]: { S: id }, + }, + ConsistentRead: true, + ProjectionExpression: '#value', + ExpressionAttributeNames: { + '#value': VALUE_ATTRIBUTE, + }, + }), + ); + + if (!result.Item) { + throw new Error(`${description} item '${scope}/${id}' was not found`); + } + + const value = result.Item[VALUE_ATTRIBUTE]?.S; + if (value === undefined) { + throw new Error(`${description} item '${scope}/${id}' does not contain a string value`); + } + + return value; +} diff --git a/lambdas/libs/storage-providers/aws/dynamodb/environment.d.ts b/lambdas/libs/storage-providers/aws/dynamodb/environment.d.ts new file mode 100644 index 0000000000..59be4dbf10 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/environment.d.ts @@ -0,0 +1,13 @@ +export {}; + +declare global { + namespace NodeJS { + interface ProcessEnv { + RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME?: string; + RUNNER_CONFIG_DYNAMODB_ENTRY_ID?: string; + RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TABLE_NAME?: string; + RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TTL_SECONDS?: string; + RUNNER_CONFIG_DYNAMODB_TTL_SECONDS?: string; + } + } +} diff --git a/lambdas/libs/storage-providers/aws/dynamodb/environment.ts b/lambdas/libs/storage-providers/aws/dynamodb/environment.ts new file mode 100644 index 0000000000..fc0a394139 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/environment.ts @@ -0,0 +1,29 @@ +type DynamoDbEnvironmentVariable = + | 'RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME' + | 'RUNNER_CONFIG_DYNAMODB_ENTRY_ID' + | 'RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TABLE_NAME' + | 'RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TTL_SECONDS' + | 'RUNNER_CONFIG_DYNAMODB_TTL_SECONDS'; + +export function requiredEnvironmentValue(name: DynamoDbEnvironmentVariable): string { + const value = process.env[name]?.trim(); + if (!value) { + throw new Error(`Environment variable ${name} is not set`); + } + + return value; +} + +export function positiveIntegerEnvironmentValue(name: DynamoDbEnvironmentVariable): number { + const value = requiredEnvironmentValue(name); + if (!/^[1-9]\d*$/.test(value)) { + throw new Error(`Environment variable ${name} must be a positive integer`); + } + + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) { + throw new Error(`Environment variable ${name} must be a positive integer`); + } + + return parsed; +} diff --git a/lambdas/libs/storage-providers/aws/dynamodb/github-app-credentials-store.test.ts b/lambdas/libs/storage-providers/aws/dynamodb/github-app-credentials-store.test.ts new file mode 100644 index 0000000000..500ad90749 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/github-app-credentials-store.test.ts @@ -0,0 +1,103 @@ +import { DynamoDBClient, GetItemCommand } from '@aws-sdk/client-dynamodb'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest/vitest'; +import { beforeEach, describe, expect, it } from 'vitest'; + +import { resetDynamoDbClient } from './client'; +import { createAwsDynamoDbGitHubAppCredentialsStore } from './github-app-credentials-store'; + +const mockDynamoDbClient = mockClient(DynamoDBClient); +const cleanEnv = process.env; + +describe('aws_dynamodb GitHub App credentials store', () => { + beforeEach(() => { + mockDynamoDbClient.reset(); + resetDynamoDbClient(); + process.env = { ...cleanEnv }; + process.env.AWS_REGION = 'eu-west-1'; + process.env.RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME = 'runner-configuration'; + }); + + it('strongly reads and decodes an ordered credential array', async () => { + const value = JSON.stringify([ + { appId: 123, privateKeyBase64: Buffer.from('primary\\nkey').toString('base64') }, + { + appId: 456, + privateKeyBase64: Buffer.from('additional-key').toString('base64'), + installationId: 789, + }, + ]); + mockDynamoDbClient.on(GetItemCommand).resolves({ Item: { value: { S: value } } }); + const store = createAwsDynamoDbGitHubAppCredentialsStore(); + + await expect(store.get()).resolves.toEqual([ + { appId: 123, privateKey: 'primary\nkey', installationId: undefined }, + { appId: 456, privateKey: 'additional-key', installationId: 789 }, + ]); + expect(mockDynamoDbClient).toHaveReceivedCommandWith(GetItemCommand, { + TableName: 'runner-configuration', + Key: { + scope: { S: 'global#github-app' }, + id: { S: 'github-app-credentials' }, + }, + ConsistentRead: true, + ProjectionExpression: '#value', + ExpressionAttributeNames: { '#value': 'value' }, + }); + }); + + it.each([ + ['not-json', 'contains invalid JSON'], + ['[]', 'must contain a non-empty array'], + [JSON.stringify([null]), 'credential at index 0 has an invalid stored value'], + [JSON.stringify([{ appId: 0, privateKeyBase64: 'a2V5' }]), 'credential at index 0 has an invalid stored value'], + [ + JSON.stringify([{ appId: 1, privateKeyBase64: 'not-base64' }]), + 'credential at index 0 has an invalid stored value', + ], + [ + JSON.stringify([{ appId: 1, privateKeyBase64: 'a2V5', installationId: 1.5 }]), + 'credential at index 0 has an invalid stored value', + ], + ])('rejects malformed stored credentials without returning their value', async (value, message) => { + mockDynamoDbClient.on(GetItemCommand).resolves({ Item: { value: { S: value } } }); + const store = createAwsDynamoDbGitHubAppCredentialsStore(); + + await expect(store.get()).rejects.toThrow(message); + }); + + it('rejects a missing credentials item', async () => { + mockDynamoDbClient.on(GetItemCommand).resolves({}); + + await expect(createAwsDynamoDbGitHubAppCredentialsStore().get()).rejects.toThrow( + "GitHub App credentials item 'global#github-app/github-app-credentials' was not found", + ); + }); + + it('rejects a non-string credentials value', async () => { + mockDynamoDbClient.on(GetItemCommand).resolves({ Item: { value: { L: [] } } }); + + await expect(createAwsDynamoDbGitHubAppCredentialsStore().get()).rejects.toThrow( + "GitHub App credentials item 'global#github-app/github-app-credentials' does not contain a string value", + ); + }); + + it.each([undefined, '', ' '])('requires the durable table name for input %j', (tableName) => { + if (tableName === undefined) { + delete process.env.RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME; + } else { + process.env.RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME = tableName; + } + + expect(() => createAwsDynamoDbGitHubAppCredentialsStore()).toThrow( + 'Environment variable RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME is not set', + ); + }); + + it('propagates reads errors without exposing stored credentials', async () => { + const error = new Error('access denied'); + mockDynamoDbClient.on(GetItemCommand).rejects(error); + + await expect(createAwsDynamoDbGitHubAppCredentialsStore().get()).rejects.toBe(error); + }); +}); diff --git a/lambdas/libs/storage-providers/aws/dynamodb/github-app-credentials-store.ts b/lambdas/libs/storage-providers/aws/dynamodb/github-app-credentials-store.ts new file mode 100644 index 0000000000..e9b1a56c18 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/github-app-credentials-store.ts @@ -0,0 +1,89 @@ +import type { GitHubAppCredential, GitHubAppCredentialsStore } from '../../core'; +import { getDurableConfigValue } from './durable-config'; +import { requiredEnvironmentValue } from './environment'; +import { GITHUB_APP_CREDENTIALS_ID, GITHUB_APP_SCOPE } from './keys'; + +interface StoredGitHubAppCredential { + appId: number; + privateKeyBase64: string; + installationId?: number; +} + +export function createAwsDynamoDbGitHubAppCredentialsStore(): GitHubAppCredentialsStore { + return new AwsDynamoDbGitHubAppCredentialsStore(requiredEnvironmentValue('RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME')); +} + +class AwsDynamoDbGitHubAppCredentialsStore implements GitHubAppCredentialsStore { + constructor(private readonly tableName: string) {} + + async get(): Promise { + const value = await getDurableConfigValue( + this.tableName, + GITHUB_APP_SCOPE, + GITHUB_APP_CREDENTIALS_ID, + 'GitHub App credentials', + ); + const credentials = parseCredentials(value); + + return credentials.map((credential) => ({ + appId: credential.appId, + privateKey: decodePrivateKey(credential.privateKeyBase64), + installationId: credential.installationId, + })); + } +} + +function parseCredentials(value: string): StoredGitHubAppCredential[] { + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + throw new Error('GitHub App credentials item contains invalid JSON'); + } + + if (!Array.isArray(parsed) || parsed.length === 0) { + throw new Error('GitHub App credentials item must contain a non-empty array'); + } + + return parsed.map((credential, index) => parseCredential(credential, index)); +} + +function parseCredential(value: unknown, index: number): StoredGitHubAppCredential { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw invalidCredential(index); + } + + const credential = value as Record; + if (!isPositiveSafeInteger(credential.appId) || !isValidBase64(credential.privateKeyBase64)) { + throw invalidCredential(index); + } + if (credential.installationId !== undefined && !isPositiveSafeInteger(credential.installationId)) { + throw invalidCredential(index); + } + + return { + appId: credential.appId, + privateKeyBase64: credential.privateKeyBase64, + installationId: credential.installationId, + }; +} + +function isPositiveSafeInteger(value: unknown): value is number { + return typeof value === 'number' && Number.isSafeInteger(value) && value > 0; +} + +function isValidBase64(value: unknown): value is string { + if (typeof value !== 'string' || value.length === 0 || value.length % 4 !== 0) { + return false; + } + + return /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value); +} + +function decodePrivateKey(privateKeyBase64: string): string { + return Buffer.from(privateKeyBase64, 'base64').toString().replace(/\\n/g, '\n'); +} + +function invalidCredential(index: number): Error { + return new Error(`GitHub App credential at index ${index} has an invalid stored value`); +} diff --git a/lambdas/libs/storage-providers/aws/dynamodb/github-webhook-secret-store.test.ts b/lambdas/libs/storage-providers/aws/dynamodb/github-webhook-secret-store.test.ts new file mode 100644 index 0000000000..c66272c2c7 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/github-webhook-secret-store.test.ts @@ -0,0 +1,74 @@ +import { DynamoDBClient, GetItemCommand } from '@aws-sdk/client-dynamodb'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest/vitest'; +import { beforeEach, describe, expect, it } from 'vitest'; + +import { resetDynamoDbClient } from './client'; +import { createAwsDynamoDbGitHubWebhookSecretStore } from './github-webhook-secret-store'; + +const mockDynamoDbClient = mockClient(DynamoDBClient); +const cleanEnv = process.env; + +describe('aws_dynamodb GitHub webhook secret store', () => { + beforeEach(() => { + mockDynamoDbClient.reset(); + resetDynamoDbClient(); + process.env = { ...cleanEnv }; + process.env.AWS_REGION = 'eu-west-1'; + process.env.RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME = 'runner-configuration'; + }); + + it('strongly reads the global webhook secret', async () => { + mockDynamoDbClient.on(GetItemCommand).resolves({ Item: { value: { S: 'webhook-secret' } } }); + + await expect(createAwsDynamoDbGitHubWebhookSecretStore().get()).resolves.toBe('webhook-secret'); + expect(mockDynamoDbClient).toHaveReceivedCommandWith(GetItemCommand, { + TableName: 'runner-configuration', + Key: { + scope: { S: 'global#webhook' }, + id: { S: 'github-webhook-secret' }, + }, + ConsistentRead: true, + ProjectionExpression: '#value', + ExpressionAttributeNames: { '#value': 'value' }, + }); + }); + + it('leaves empty-value validation to the webhook config loader', async () => { + mockDynamoDbClient.on(GetItemCommand).resolves({ Item: { value: { S: '' } } }); + + await expect(createAwsDynamoDbGitHubWebhookSecretStore().get()).resolves.toBe(''); + }); + + it('rejects a missing secret item without logging or returning a value', async () => { + mockDynamoDbClient.on(GetItemCommand).resolves({}); + + await expect(createAwsDynamoDbGitHubWebhookSecretStore().get()).rejects.toThrow( + "GitHub webhook secret item 'global#webhook/github-webhook-secret' was not found", + ); + }); + + it('rejects a non-string secret item', async () => { + mockDynamoDbClient.on(GetItemCommand).resolves({ Item: { value: { B: new Uint8Array() } } }); + + await expect(createAwsDynamoDbGitHubWebhookSecretStore().get()).rejects.toThrow( + "GitHub webhook secret item 'global#webhook/github-webhook-secret' does not contain a string value", + ); + }); + + it('requires the durable table name before reading', () => { + delete process.env.RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME; + + expect(() => createAwsDynamoDbGitHubWebhookSecretStore()).toThrow( + 'Environment variable RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME is not set', + ); + expect(mockDynamoDbClient.calls()).toHaveLength(0); + }); + + it('propagates read errors without handling the secret value', async () => { + const error = new Error('access denied'); + mockDynamoDbClient.on(GetItemCommand).rejects(error); + + await expect(createAwsDynamoDbGitHubWebhookSecretStore().get()).rejects.toBe(error); + }); +}); diff --git a/lambdas/libs/storage-providers/aws/dynamodb/github-webhook-secret-store.ts b/lambdas/libs/storage-providers/aws/dynamodb/github-webhook-secret-store.ts new file mode 100644 index 0000000000..6fbf8f81b4 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/github-webhook-secret-store.ts @@ -0,0 +1,21 @@ +import type { GitHubWebhookSecretStore } from '../../core'; +import { getDurableConfigValue } from './durable-config'; +import { requiredEnvironmentValue } from './environment'; +import { GITHUB_WEBHOOK_SCOPE, GITHUB_WEBHOOK_SECRET_ID } from './keys'; + +export function createAwsDynamoDbGitHubWebhookSecretStore(): GitHubWebhookSecretStore { + return new AwsDynamoDbGitHubWebhookSecretStore(requiredEnvironmentValue('RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME')); +} + +class AwsDynamoDbGitHubWebhookSecretStore implements GitHubWebhookSecretStore { + constructor(private readonly tableName: string) {} + + async get(): Promise { + return await getDurableConfigValue( + this.tableName, + GITHUB_WEBHOOK_SCOPE, + GITHUB_WEBHOOK_SECRET_ID, + 'GitHub webhook secret', + ); + } +} diff --git a/lambdas/libs/storage-providers/aws/dynamodb/keys.test.ts b/lambdas/libs/storage-providers/aws/dynamodb/keys.test.ts new file mode 100644 index 0000000000..ac651df069 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/keys.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from 'vitest'; + +import { + GITHUB_APP_CREDENTIALS_ID, + GITHUB_APP_SCOPE, + GITHUB_WEBHOOK_SCOPE, + GITHUB_WEBHOOK_SECRET_ID, + RUNNER_BOOTSTRAP_CONFIG_ID, + RUNNER_CONFIG_ID, + RUNNER_MATCHER_CONFIG_ID, + RUNNER_MATCHER_SCOPE, + runnerBootstrapScope, + runnerGroupId, + runnerGroupScope, + runnerStateId, + runnerStateScope, +} from './keys'; + +describe('aws_dynamodb storage keys', () => { + it('isolates whole-deployment durable records by capability', () => { + expect({ scope: GITHUB_APP_SCOPE, id: GITHUB_APP_CREDENTIALS_ID }).toEqual({ + scope: 'global#github-app', + id: 'github-app-credentials', + }); + expect({ scope: GITHUB_WEBHOOK_SCOPE, id: GITHUB_WEBHOOK_SECRET_ID }).toEqual({ + scope: 'global#webhook', + id: 'github-webhook-secret', + }); + expect({ scope: RUNNER_MATCHER_SCOPE, id: RUNNER_MATCHER_CONFIG_ID }).toEqual({ + scope: 'global#matcher', + id: 'runner-matcher-config', + }); + }); + + it('isolates entry records by access boundary', () => { + expect({ scope: runnerBootstrapScope('linux-x64'), id: RUNNER_BOOTSTRAP_CONFIG_ID }).toEqual({ + scope: 'entry#linux-x64#bootstrap', + id: 'runner-config', + }); + expect({ scope: runnerGroupScope('linux-x64'), id: runnerGroupId('Default') }).toEqual({ + scope: 'entry#linux-x64#runner-group', + id: 'runner-group#Default', + }); + expect(RUNNER_CONFIG_ID).toBe('config'); + expect({ scope: runnerStateScope('linux-x64'), id: runnerStateId('runner-123') }).toEqual({ + scope: 'entry#linux-x64#runner-state', + id: 'runner#runner-123', + }); + }); +}); diff --git a/lambdas/libs/storage-providers/aws/dynamodb/keys.ts b/lambdas/libs/storage-providers/aws/dynamodb/keys.ts new file mode 100644 index 0000000000..cef92af6bb --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/keys.ts @@ -0,0 +1,34 @@ +export const SCOPE_ATTRIBUTE = 'scope'; +export const ID_ATTRIBUTE = 'id'; +export const VALUE_ATTRIBUTE = 'value'; +export const EXPIRES_AT_ATTRIBUTE = 'expires_at'; + +export const GITHUB_APP_SCOPE = 'global#github-app'; +export const GITHUB_WEBHOOK_SCOPE = 'global#webhook'; +export const RUNNER_MATCHER_SCOPE = 'global#matcher'; + +export const GITHUB_APP_CREDENTIALS_ID = 'github-app-credentials'; +export const GITHUB_WEBHOOK_SECRET_ID = 'github-webhook-secret'; +export const RUNNER_MATCHER_CONFIG_ID = 'runner-matcher-config'; +export const RUNNER_BOOTSTRAP_CONFIG_ID = 'runner-config'; +export const RUNNER_CONFIG_ID = 'config'; + +export function runnerBootstrapScope(entryId: string): string { + return `entry#${entryId}#bootstrap`; +} + +export function runnerGroupScope(entryId: string): string { + return `entry#${entryId}#runner-group`; +} + +export function runnerStateScope(entryId: string): string { + return `entry#${entryId}#runner-state`; +} + +export function runnerStateId(runnerId: string): string { + return `runner#${runnerId}`; +} + +export function runnerGroupId(runnerGroupName: string): string { + return `runner-group#${runnerGroupName}`; +} diff --git a/lambdas/libs/storage-providers/aws/dynamodb/runner-config-store.test.ts b/lambdas/libs/storage-providers/aws/dynamodb/runner-config-store.test.ts new file mode 100644 index 0000000000..1afb6c7fe9 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/runner-config-store.test.ts @@ -0,0 +1,148 @@ +import { DynamoDBClient, PutItemCommand } from '@aws-sdk/client-dynamodb'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest/vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { resetDynamoDbClient } from './client'; +import { createAwsDynamoDbRunnerConfigStore } from './runner-config-store'; + +const mockDynamoDbClient = mockClient(DynamoDBClient); +const cleanEnv = process.env; + +describe('aws_dynamodb runner config store', () => { + beforeEach(() => { + mockDynamoDbClient.reset(); + resetDynamoDbClient(); + process.env = { ...cleanEnv }; + process.env.AWS_REGION = 'eu-west-1'; + process.env.RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TABLE_NAME = 'runner-state'; + delete process.env.RUNNER_CONFIG_DYNAMODB_ENTRY_ID; + process.env.RUNNER_CONFIG_DYNAMODB_TTL_SECONDS = '3600'; + vi.useFakeTimers(); + vi.setSystemTime(new Date('2025-01-01T00:00:00.000Z')); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it('creates an expiring runner config without overwriting an existing record', async () => { + const store = createAwsDynamoDbRunnerConfigStore(); + + await store.create({ + runnerId: 'runner-123', + value: 'encoded-jit-config', + accessScope: 'arn:aws:ec2:eu-west-1:123456789012:instance/i-123', + }); + + expect(store.maxWritesPerSecond).toBeUndefined(); + expect(mockDynamoDbClient).toHaveReceivedCommandWith(PutItemCommand, { + TableName: 'runner-state', + Item: { + scope: { S: 'arn:aws:ec2:eu-west-1:123456789012:instance/i-123' }, + id: { S: 'config' }, + value: { S: 'encoded-jit-config' }, + expires_at: { N: '1735693200' }, + }, + ConditionExpression: 'attribute_not_exists(#scope) AND attribute_not_exists(#id)', + ExpressionAttributeNames: { + '#scope': 'scope', + '#id': 'id', + }, + }); + }); + + it('stores provider-neutral metadata without exposing it as top-level attributes', async () => { + const store = createAwsDynamoDbRunnerConfigStore(); + + await store.create( + { + runnerId: 'runner-123', + value: 'registration-config', + accessScope: 'arn:aws:ec2:eu-west-1:123456789012:instance/i-123', + }, + { + metadata: [ + { key: 'InstanceId', value: 'i-123' }, + { key: 'Environment', value: 'test' }, + ], + }, + ); + + const command = mockDynamoDbClient.commandCalls(PutItemCommand)[0].args[0]; + expect(command.input.Item?.metadata).toEqual({ + L: [ + { M: { key: { S: 'InstanceId' }, value: { S: 'i-123' } } }, + { M: { key: { S: 'Environment' }, value: { S: 'test' } } }, + ], + }); + }); + + it('does not write metadata for an empty metadata list', async () => { + const store = createAwsDynamoDbRunnerConfigStore(); + + await store.create( + { + runnerId: 'runner-123', + value: 'registration-config', + accessScope: 'arn:aws:ec2:eu-west-1:123456789012:instance/i-123', + }, + { metadata: [] }, + ); + + const command = mockDynamoDbClient.commandCalls(PutItemCommand)[0].args[0]; + expect(command.input.Item).not.toHaveProperty('metadata'); + }); + + it('relies on DynamoDB TTL instead of scanning or deleting during housekeeping', async () => { + const store = createAwsDynamoDbRunnerConfigStore(); + + await expect(store.houseKeeper()).resolves.toBeUndefined(); + + expect(mockDynamoDbClient.calls()).toHaveLength(0); + }); + + it.each(['RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TABLE_NAME', 'RUNNER_CONFIG_DYNAMODB_TTL_SECONDS'] as const)( + 'rejects a missing or blank %s', + (name) => { + delete process.env[name]; + expect(() => createAwsDynamoDbRunnerConfigStore()).toThrow(`Environment variable ${name} is not set`); + + process.env[name] = ' '; + expect(() => createAwsDynamoDbRunnerConfigStore()).toThrow(`Environment variable ${name} is not set`); + expect(mockDynamoDbClient.calls()).toHaveLength(0); + }, + ); + + it.each(['0', '-1', '1.5', 'not-a-number', '9007199254740992'])('rejects invalid TTL seconds %j', (ttlSeconds) => { + process.env.RUNNER_CONFIG_DYNAMODB_TTL_SECONDS = ttlSeconds; + + expect(() => createAwsDynamoDbRunnerConfigStore()).toThrow( + 'Environment variable RUNNER_CONFIG_DYNAMODB_TTL_SECONDS must be a positive integer', + ); + expect(mockDynamoDbClient.calls()).toHaveLength(0); + }); + + it.each([undefined, '', ' '])('rejects invalid access scope %j before writing', async (accessScope) => { + const store = createAwsDynamoDbRunnerConfigStore(); + + await expect(store.create({ runnerId: 'runner-123', value: 'sensitive-config', accessScope })).rejects.toThrow( + "Runner config field 'accessScope' must be a non-empty string for aws_dynamodb", + ); + expect(mockDynamoDbClient.calls()).toHaveLength(0); + }); + + it('propagates DynamoDB write errors without handling the stored value', async () => { + const error = new Error('conditional request failed'); + mockDynamoDbClient.on(PutItemCommand).rejects(error); + const store = createAwsDynamoDbRunnerConfigStore(); + + await expect( + store.create({ + runnerId: 'runner-123', + value: 'sensitive-config', + accessScope: 'arn:aws:ec2:eu-west-1:123456789012:instance/i-123', + }), + ).rejects.toBe(error); + }); +}); diff --git a/lambdas/libs/storage-providers/aws/dynamodb/runner-config-store.ts b/lambdas/libs/storage-providers/aws/dynamodb/runner-config-store.ts new file mode 100644 index 0000000000..79de2b6b71 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/runner-config-store.ts @@ -0,0 +1,66 @@ +import { PutItemCommand, type AttributeValue } from '@aws-sdk/client-dynamodb'; + +import type { RunnerConfigMetadata, RunnerConfigRecord, RunnerConfigStore } from '../../core'; +import { getDynamoDbClient } from './client'; +import { positiveIntegerEnvironmentValue, requiredEnvironmentValue } from './environment'; +import { EXPIRES_AT_ATTRIBUTE, ID_ATTRIBUTE, RUNNER_CONFIG_ID, SCOPE_ATTRIBUTE, VALUE_ATTRIBUTE } from './keys'; + +const METADATA_ATTRIBUTE = 'metadata'; + +interface AwsDynamoDbRunnerConfigStoreConfig { + tableName: string; + ttlSeconds: number; +} + +export function createAwsDynamoDbRunnerConfigStore(): RunnerConfigStore { + return new AwsDynamoDbRunnerConfigStore({ + tableName: requiredEnvironmentValue('RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TABLE_NAME'), + ttlSeconds: positiveIntegerEnvironmentValue('RUNNER_CONFIG_DYNAMODB_TTL_SECONDS'), + }); +} + +class AwsDynamoDbRunnerConfigStore implements RunnerConfigStore { + constructor(private readonly config: AwsDynamoDbRunnerConfigStoreConfig) {} + + async create(record: RunnerConfigRecord, options: { metadata?: RunnerConfigMetadata[] } = {}): Promise { + if (typeof record.accessScope !== 'string' || record.accessScope.trim() === '') { + throw new Error("Runner config field 'accessScope' must be a non-empty string for aws_dynamodb"); + } + + const item: Record = { + [SCOPE_ATTRIBUTE]: { S: record.accessScope }, + [ID_ATTRIBUTE]: { S: RUNNER_CONFIG_ID }, + [VALUE_ATTRIBUTE]: { S: record.value }, + [EXPIRES_AT_ATTRIBUTE]: { + N: (Math.floor(Date.now() / 1000) + this.config.ttlSeconds).toString(), + }, + }; + + if (options.metadata && options.metadata.length > 0) { + item[METADATA_ATTRIBUTE] = { + L: options.metadata.map(({ key, value }) => ({ + M: { + key: { S: key }, + value: { S: value }, + }, + })), + }; + } + + await getDynamoDbClient().send( + new PutItemCommand({ + TableName: this.config.tableName, + Item: item, + ConditionExpression: 'attribute_not_exists(#scope) AND attribute_not_exists(#id)', + ExpressionAttributeNames: { + '#scope': SCOPE_ATTRIBUTE, + '#id': ID_ATTRIBUTE, + }, + }), + ); + } + + async houseKeeper(): Promise { + // DynamoDB TTL removes expired runner config records without a scan/delete job. + } +} diff --git a/lambdas/libs/storage-providers/aws/dynamodb/runner-group-cache-store.test.ts b/lambdas/libs/storage-providers/aws/dynamodb/runner-group-cache-store.test.ts new file mode 100644 index 0000000000..2fbac5b69d --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/runner-group-cache-store.test.ts @@ -0,0 +1,108 @@ +import { DynamoDBClient, GetItemCommand, PutItemCommand } from '@aws-sdk/client-dynamodb'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest/vitest'; +import { beforeEach, describe, expect, it } from 'vitest'; + +import { resetDynamoDbClient } from './client'; +import { createAwsDynamoDbRunnerGroupCacheStore } from './runner-group-cache-store'; + +const mockDynamoDbClient = mockClient(DynamoDBClient); +const cleanEnv = process.env; + +describe('aws_dynamodb runner group cache store', () => { + beforeEach(() => { + mockDynamoDbClient.reset(); + resetDynamoDbClient(); + process.env = { ...cleanEnv }; + process.env.AWS_REGION = 'eu-west-1'; + process.env.RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME = 'runner-configuration'; + process.env.RUNNER_CONFIG_DYNAMODB_ENTRY_ID = 'linux-x64'; + }); + + it('gets a runner group id with a strongly consistent projected read', async () => { + mockDynamoDbClient.on(GetItemCommand).resolves({ Item: { value: { S: '42' } } }); + const store = createAwsDynamoDbRunnerGroupCacheStore(); + + await expect(store.get('Default')).resolves.toBe(42); + expect(mockDynamoDbClient).toHaveReceivedCommandWith(GetItemCommand, { + TableName: 'runner-configuration', + Key: { + scope: { S: 'entry#linux-x64#runner-group' }, + id: { S: 'runner-group#Default' }, + }, + ConsistentRead: true, + ProjectionExpression: '#value', + ExpressionAttributeNames: { + '#value': 'value', + }, + }); + }); + + it('returns undefined when the runner group is not cached', async () => { + mockDynamoDbClient.on(GetItemCommand).resolves({}); + const store = createAwsDynamoDbRunnerGroupCacheStore(); + + await expect(store.get('Default')).resolves.toBeUndefined(); + }); + + it.each([ + [{ value: { N: '42' } }, 'non-string value'], + [{ value: { S: '42cached' } }, 'partially numeric value'], + [{ value: { S: '9007199254740992' } }, 'unsafe integer value'], + ])('rejects an invalid cached runner group id: %s', async (item) => { + mockDynamoDbClient.on(GetItemCommand).resolves({ Item: item }); + const store = createAwsDynamoDbRunnerGroupCacheStore(); + + await expect(store.get('Default')).rejects.toThrow( + "Runner group cache item 'entry#linux-x64#runner-group/runner-group#Default' has an invalid value", + ); + }); + + it('creates a runner group cache record without overwriting an existing record', async () => { + const store = createAwsDynamoDbRunnerGroupCacheStore(); + + await store.create({ runnerGroupName: 'Default', runnerGroupId: 42 }); + + expect(mockDynamoDbClient).toHaveReceivedCommandWith(PutItemCommand, { + TableName: 'runner-configuration', + Item: { + scope: { S: 'entry#linux-x64#runner-group' }, + id: { S: 'runner-group#Default' }, + value: { S: '42' }, + }, + ConditionExpression: 'attribute_not_exists(#scope) AND attribute_not_exists(#id)', + ExpressionAttributeNames: { + '#scope': 'scope', + '#id': 'id', + }, + }); + }); + + it.each(['RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME', 'RUNNER_CONFIG_DYNAMODB_ENTRY_ID'] as const)( + 'rejects a missing or blank %s', + (name) => { + delete process.env[name]; + expect(() => createAwsDynamoDbRunnerGroupCacheStore()).toThrow(`Environment variable ${name} is not set`); + + process.env[name] = ' '; + expect(() => createAwsDynamoDbRunnerGroupCacheStore()).toThrow(`Environment variable ${name} is not set`); + expect(mockDynamoDbClient.calls()).toHaveLength(0); + }, + ); + + it('propagates DynamoDB read errors', async () => { + const error = new Error('read failed'); + mockDynamoDbClient.on(GetItemCommand).rejects(error); + const store = createAwsDynamoDbRunnerGroupCacheStore(); + + await expect(store.get('Default')).rejects.toBe(error); + }); + + it('propagates DynamoDB write errors', async () => { + const error = new Error('conditional request failed'); + mockDynamoDbClient.on(PutItemCommand).rejects(error); + const store = createAwsDynamoDbRunnerGroupCacheStore(); + + await expect(store.create({ runnerGroupName: 'Default', runnerGroupId: 42 })).rejects.toBe(error); + }); +}); diff --git a/lambdas/libs/storage-providers/aws/dynamodb/runner-group-cache-store.ts b/lambdas/libs/storage-providers/aws/dynamodb/runner-group-cache-store.ts new file mode 100644 index 0000000000..a90a79e46e --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/runner-group-cache-store.ts @@ -0,0 +1,80 @@ +import { GetItemCommand, PutItemCommand } from '@aws-sdk/client-dynamodb'; + +import type { RunnerGroupCacheRecord, RunnerGroupCacheStore } from '../../core'; +import { getDynamoDbClient } from './client'; +import { requiredEnvironmentValue } from './environment'; +import { + ID_ATTRIBUTE, + runnerGroupId as runnerGroupItemId, + runnerGroupScope, + SCOPE_ATTRIBUTE, + VALUE_ATTRIBUTE, +} from './keys'; + +interface AwsDynamoDbRunnerGroupCacheStoreConfig { + tableName: string; + scope: string; +} + +export function createAwsDynamoDbRunnerGroupCacheStore(): RunnerGroupCacheStore { + return new AwsDynamoDbRunnerGroupCacheStore({ + tableName: requiredEnvironmentValue('RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME'), + scope: runnerGroupScope(requiredEnvironmentValue('RUNNER_CONFIG_DYNAMODB_ENTRY_ID')), + }); +} + +class AwsDynamoDbRunnerGroupCacheStore implements RunnerGroupCacheStore { + constructor(private readonly config: AwsDynamoDbRunnerGroupCacheStoreConfig) {} + + async get(runnerGroupName: string): Promise { + const id = runnerGroupItemId(runnerGroupName); + const result = await getDynamoDbClient().send( + new GetItemCommand({ + TableName: this.config.tableName, + Key: { + [SCOPE_ATTRIBUTE]: { S: this.config.scope }, + [ID_ATTRIBUTE]: { S: id }, + }, + ConsistentRead: true, + ProjectionExpression: '#value', + ExpressionAttributeNames: { + '#value': VALUE_ATTRIBUTE, + }, + }), + ); + + if (!result.Item) { + return undefined; + } + + const value = result.Item[VALUE_ATTRIBUTE]?.S; + if (value === undefined || !/^\d+$/.test(value)) { + throw new Error(`Runner group cache item '${this.config.scope}/${id}' has an invalid value`); + } + + const runnerGroupId = Number(value); + if (!Number.isSafeInteger(runnerGroupId)) { + throw new Error(`Runner group cache item '${this.config.scope}/${id}' has an invalid value`); + } + + return runnerGroupId; + } + + async create(record: RunnerGroupCacheRecord): Promise { + await getDynamoDbClient().send( + new PutItemCommand({ + TableName: this.config.tableName, + Item: { + [SCOPE_ATTRIBUTE]: { S: this.config.scope }, + [ID_ATTRIBUTE]: { S: runnerGroupItemId(record.runnerGroupName) }, + [VALUE_ATTRIBUTE]: { S: record.runnerGroupId.toString() }, + }, + ConditionExpression: 'attribute_not_exists(#scope) AND attribute_not_exists(#id)', + ExpressionAttributeNames: { + '#scope': SCOPE_ATTRIBUTE, + '#id': ID_ATTRIBUTE, + }, + }), + ); + } +} diff --git a/lambdas/libs/storage-providers/aws/dynamodb/runner-matcher-config-store.test.ts b/lambdas/libs/storage-providers/aws/dynamodb/runner-matcher-config-store.test.ts new file mode 100644 index 0000000000..66e8d07c2e --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/runner-matcher-config-store.test.ts @@ -0,0 +1,95 @@ +import { DynamoDBClient, GetItemCommand } from '@aws-sdk/client-dynamodb'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest/vitest'; +import { beforeEach, describe, expect, it } from 'vitest'; + +import { resetDynamoDbClient } from './client'; +import { createAwsDynamoDbRunnerMatcherConfigStore } from './runner-matcher-config-store'; + +const mockDynamoDbClient = mockClient(DynamoDBClient); +const cleanEnv = process.env; + +describe('aws_dynamodb runner matcher config store', () => { + beforeEach(() => { + mockDynamoDbClient.reset(); + resetDynamoDbClient(); + process.env = { ...cleanEnv }; + process.env.AWS_REGION = 'eu-west-1'; + process.env.RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME = 'runner-configuration'; + }); + + it('gets the matcher config with a strongly consistent projected read', async () => { + mockDynamoDbClient.on(GetItemCommand).resolves({ Item: { value: { S: '[{"id":"runner"}]' } } }); + const store = createAwsDynamoDbRunnerMatcherConfigStore(); + + await expect(store.get()).resolves.toBe('[{"id":"runner"}]'); + expect(mockDynamoDbClient).toHaveReceivedCommandWith(GetItemCommand, { + TableName: 'runner-configuration', + Key: { + scope: { S: 'global#matcher' }, + id: { S: 'runner-matcher-config' }, + }, + ConsistentRead: true, + ProjectionExpression: '#value', + ExpressionAttributeNames: { + '#value': 'value', + }, + }); + }); + + it('returns an empty stored string for validation by the webhook config loader', async () => { + mockDynamoDbClient.on(GetItemCommand).resolves({ Item: { value: { S: '' } } }); + const store = createAwsDynamoDbRunnerMatcherConfigStore(); + + await expect(store.get()).resolves.toBe(''); + }); + + it('rejects a missing matcher config item', async () => { + mockDynamoDbClient.on(GetItemCommand).resolves({}); + const store = createAwsDynamoDbRunnerMatcherConfigStore(); + + await expect(store.get()).rejects.toThrow( + "Runner matcher config item 'global#matcher/runner-matcher-config' was not found", + ); + }); + + it('rejects a matcher config item without a string value', async () => { + mockDynamoDbClient.on(GetItemCommand).resolves({ Item: { value: { N: '1' } } }); + const store = createAwsDynamoDbRunnerMatcherConfigStore(); + + await expect(store.get()).rejects.toThrow( + "Runner matcher config item 'global#matcher/runner-matcher-config' does not contain a string value", + ); + }); + + it('rejects a missing or blank durable table name', () => { + delete process.env.RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME; + expect(() => createAwsDynamoDbRunnerMatcherConfigStore()).toThrow( + 'Environment variable RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME is not set', + ); + + process.env.RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME = ' '; + expect(() => createAwsDynamoDbRunnerMatcherConfigStore()).toThrow( + 'Environment variable RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME is not set', + ); + expect(mockDynamoDbClient.calls()).toHaveLength(0); + }); + + it('propagates DynamoDB read errors', async () => { + const error = new Error('read failed'); + mockDynamoDbClient.on(GetItemCommand).rejects(error); + const store = createAwsDynamoDbRunnerMatcherConfigStore(); + + await expect(store.get()).rejects.toBe(error); + }); + + it('reuses the memoised client across reads', async () => { + mockDynamoDbClient.on(GetItemCommand).resolves({ Item: { value: { S: '[]' } } }); + const store = createAwsDynamoDbRunnerMatcherConfigStore(); + + await store.get(); + await store.get(); + + expect(mockDynamoDbClient.commandCalls(GetItemCommand)).toHaveLength(2); + }); +}); diff --git a/lambdas/libs/storage-providers/aws/dynamodb/runner-matcher-config-store.ts b/lambdas/libs/storage-providers/aws/dynamodb/runner-matcher-config-store.ts new file mode 100644 index 0000000000..f840345983 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/runner-matcher-config-store.ts @@ -0,0 +1,27 @@ +import type { RunnerMatcherConfigStore } from '../../core'; +import { getDurableConfigValue } from './durable-config'; +import { requiredEnvironmentValue } from './environment'; +import { RUNNER_MATCHER_CONFIG_ID, RUNNER_MATCHER_SCOPE } from './keys'; + +interface AwsDynamoDbRunnerMatcherConfigStoreConfig { + tableName: string; +} + +export function createAwsDynamoDbRunnerMatcherConfigStore(): RunnerMatcherConfigStore { + return new AwsDynamoDbRunnerMatcherConfigStore({ + tableName: requiredEnvironmentValue('RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME'), + }); +} + +class AwsDynamoDbRunnerMatcherConfigStore implements RunnerMatcherConfigStore { + constructor(private readonly config: AwsDynamoDbRunnerMatcherConfigStoreConfig) {} + + async get(): Promise { + return await getDurableConfigValue( + this.config.tableName, + RUNNER_MATCHER_SCOPE, + RUNNER_MATCHER_CONFIG_ID, + 'Runner matcher config', + ); + } +} diff --git a/lambdas/libs/storage-providers/aws/dynamodb/runner-state-store.test.ts b/lambdas/libs/storage-providers/aws/dynamodb/runner-state-store.test.ts new file mode 100644 index 0000000000..75e74ab526 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/runner-state-store.test.ts @@ -0,0 +1,451 @@ +import { + ConditionalCheckFailedException, + DeleteItemCommand, + DynamoDBClient, + PutItemCommand, + QueryCommand, + UpdateItemCommand, + type AttributeValue, +} from '@aws-sdk/client-dynamodb'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest/vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { CreateRunnerStateRecord } from '../../core'; +import { resetDynamoDbClient } from './client'; +import { createAwsDynamoDbRunnerStateStore } from './runner-state-store'; + +const mockDynamoDbClient = mockClient(DynamoDBClient); +const cleanEnv = process.env; + +describe('aws_dynamodb runner state store', () => { + beforeEach(() => { + mockDynamoDbClient.reset(); + resetDynamoDbClient(); + process.env = { ...cleanEnv }; + process.env.AWS_REGION = 'eu-west-1'; + process.env.RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TABLE_NAME = 'runner-state'; + process.env.RUNNER_CONFIG_DYNAMODB_ENTRY_ID = 'linux-x64'; + process.env.RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TTL_SECONDS = '86400'; + mockDynamoDbClient.on(PutItemCommand).resolves({}); + mockDynamoDbClient.on(UpdateItemCommand).resolves({}); + mockDynamoDbClient.on(DeleteItemCommand).resolves({}); + vi.useFakeTimers(); + vi.setSystemTime(new Date('2025-01-01T00:00:00.000Z')); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it('creates a provisioning record without a secret payload or overwrite', async () => { + const store = createAwsDynamoDbRunnerStateStore(); + + await store.create(createRecord()); + + expect(mockDynamoDbClient).toHaveReceivedCommandWith(PutItemCommand, { + TableName: 'runner-state', + Item: { + scope: { S: 'entry#linux-x64#runner-state' }, + id: { S: 'runner#runner-123' }, + runner_id: { S: 'runner-123' }, + compute_provider: { S: 'aws_ec2' }, + compute_resource_id: { S: 'i-123' }, + runner_name: { S: 'ghr-runner-123' }, + runner_labels: { L: [{ S: 'linux' }, { S: 'x64' }] }, + runner_owner: { S: 'github-aws-runners' }, + runner_type: { S: 'Org' }, + state: { S: 'provisioning' }, + created_at: { S: '2025-01-01T00:00:00.000Z' }, + updated_at: { S: '2025-01-01T00:00:00.000Z' }, + expires_at: { N: '1735776000' }, + metadata: { + L: [{ M: { key: { S: 'Environment' }, value: { S: 'test' } } }], + }, + }, + ConditionExpression: 'attribute_not_exists(#scope) AND attribute_not_exists(#id)', + ExpressionAttributeNames: { '#scope': 'scope', '#id': 'id' }, + }); + const item = mockDynamoDbClient.commandCalls(PutItemCommand)[0].args[0].input.Item; + expect(item).not.toHaveProperty('value'); + }); + + it('omits optional attributes when provisioning data is not available yet', async () => { + const store = createAwsDynamoDbRunnerStateStore(); + const record = createRecord(); + delete record.runnerName; + delete record.runnerLabels; + delete record.metadata; + + await store.create(record); + + const item = mockDynamoDbClient.commandCalls(PutItemCommand)[0].args[0].input.Item; + expect(item).not.toHaveProperty('runner_name'); + expect(item).not.toHaveProperty('runner_labels'); + expect(item).not.toHaveProperty('metadata'); + }); + + it('preserves empty provider-neutral metadata values', async () => { + const store = createAwsDynamoDbRunnerStateStore(); + + await store.create({ ...createRecord(), metadata: [{ key: 'OptionalTag', value: '' }] }); + + expect(mockDynamoDbClient.commandCalls(PutItemCommand)[0].args[0].input.Item?.metadata).toEqual({ + L: [{ M: { key: { S: 'OptionalTag' }, value: { S: '' } } }], + }); + }); + + it('activates only provisioning records and atomically adds GitHub identity metadata', async () => { + const store = createAwsDynamoDbRunnerStateStore(); + + await store.activate('runner-123', { + githubRunnerId: '9876', + runnerName: 'jit-runner', + runnerLabels: ['linux', 'arm64'], + metadata: [{ key: 'zone', value: 'eu-west-1a' }], + }); + + expect(mockDynamoDbClient).toHaveReceivedCommandWith(UpdateItemCommand, { + TableName: 'runner-state', + Key: { scope: { S: 'entry#linux-x64#runner-state' }, id: { S: 'runner#runner-123' } }, + UpdateExpression: + 'SET #state = :state, #updatedAt = :updatedAt, #runnerName = :runnerName, #runnerLabels = :runnerLabels, #githubRunnerId = :githubRunnerId, #metadata = :metadata REMOVE #expiresAt', + ConditionExpression: 'attribute_exists(#scope) AND attribute_exists(#id) AND #state IN (:expectedState0)', + ExpressionAttributeNames: { + '#scope': 'scope', + '#id': 'id', + '#state': 'state', + '#updatedAt': 'updated_at', + '#expiresAt': 'expires_at', + '#runnerName': 'runner_name', + '#runnerLabels': 'runner_labels', + '#githubRunnerId': 'github_runner_id', + '#metadata': 'metadata', + }, + ExpressionAttributeValues: { + ':state': { S: 'active' }, + ':updatedAt': { S: '2025-01-01T00:00:00.000Z' }, + ':runnerName': { S: 'jit-runner' }, + ':runnerLabels': { L: [{ S: 'linux' }, { S: 'arm64' }] }, + ':githubRunnerId': { S: '9876' }, + ':metadata': { L: [{ M: { key: { S: 'zone' }, value: { S: 'eu-west-1a' } } }] }, + ':expectedState0': { S: 'provisioning' }, + }, + }); + }); + + it('records GitHub identity while the runner remains provisioning', async () => { + const store = createAwsDynamoDbRunnerStateStore(); + + await store.recordGitHubIdentity('runner-123', { + githubRunnerId: '9876', + runnerName: 'jit-runner', + runnerLabels: ['linux', 'arm64'], + }); + + expect(mockDynamoDbClient).toHaveReceivedCommandWith(UpdateItemCommand, { + TableName: 'runner-state', + Key: { scope: { S: 'entry#linux-x64#runner-state' }, id: { S: 'runner#runner-123' } }, + UpdateExpression: + 'SET #state = :state, #updatedAt = :updatedAt, #expiresAt = :expiresAt, #runnerName = :runnerName, #runnerLabels = :runnerLabels, #githubRunnerId = :githubRunnerId', + ConditionExpression: 'attribute_exists(#scope) AND attribute_exists(#id) AND #state IN (:expectedState0)', + ExpressionAttributeValues: expect.objectContaining({ + ':state': { S: 'provisioning' }, + ':updatedAt': { S: '2025-01-01T00:00:00.000Z' }, + ':expiresAt': { N: '1735776000' }, + ':runnerName': { S: 'jit-runner' }, + ':runnerLabels': { L: [{ S: 'linux' }, { S: 'arm64' }] }, + ':githubRunnerId': { S: '9876' }, + ':expectedState0': { S: 'provisioning' }, + }), + }); + }); + + it('lists all pages for an entry with a strongly consistent query', async () => { + const lastKey = { scope: { S: 'entry#linux-x64#runner-state' }, id: { S: 'runner#runner-123' } }; + mockDynamoDbClient + .on(QueryCommand) + .resolvesOnce({ Items: [storedRecord()], LastEvaluatedKey: lastKey }) + .resolvesOnce({ Items: [storedRecord({ runnerId: 'runner-456', resourceId: 'vm-456' })] }); + const store = createAwsDynamoDbRunnerStateStore(); + + await expect(store.list()).resolves.toEqual([ + expectedRecord(), + expectedRecord({ runnerId: 'runner-456', resourceId: 'vm-456' }), + ]); + const calls = mockDynamoDbClient.commandCalls(QueryCommand); + expect(calls).toHaveLength(2); + expect(calls[0].args[0].input).toMatchObject({ + TableName: 'runner-state', + KeyConditionExpression: '#scope = :scope AND begins_with(#id, :runner)', + ConsistentRead: true, + ExpressionAttributeValues: { + ':scope': { S: 'entry#linux-x64#runner-state' }, + ':runner': { S: 'runner#' }, + }, + }); + expect(calls[1].args[0].input.ExclusiveStartKey).toEqual(lastKey); + }); + + it('filters by compute provider while retaining an entry-scoped key query', async () => { + mockDynamoDbClient.on(QueryCommand).resolves({ Items: [] }); + const store = createAwsDynamoDbRunnerStateStore(); + + await store.list({ computeProvider: 'aws_microvm' }); + + expect(mockDynamoDbClient).toHaveReceivedCommandWith(QueryCommand, { + FilterExpression: '#computeProvider = :computeProvider', + ExpressionAttributeNames: { + '#scope': 'scope', + '#id': 'id', + '#computeProvider': 'compute_provider', + }, + ExpressionAttributeValues: { + ':scope': { S: 'entry#linux-x64#runner-state' }, + ':runner': { S: 'runner#' }, + ':computeProvider': { S: 'aws_microvm' }, + }, + }); + }); + + it('maps optional fields as absent and rejects corrupt lifecycle state', async () => { + const item = storedRecord(); + delete item.runner_name; + delete item.runner_labels; + delete item.github_runner_id; + delete item.metadata; + mockDynamoDbClient.on(QueryCommand).resolves({ Items: [item] }); + const store = createAwsDynamoDbRunnerStateStore(); + + await expect(store.list()).resolves.toEqual([ + expect.objectContaining({ + runnerName: undefined, + runnerLabels: undefined, + githubRunnerId: undefined, + metadata: undefined, + }), + ]); + + item.state = { S: 'unknown' }; + mockDynamoDbClient.on(QueryCommand).resolves({ Items: [item] }); + await expect(store.list()).rejects.toThrow( + "Runner state item 'entry#linux-x64#runner-state/runner#runner-123' has an invalid 'state' attribute", + ); + }); + + it('marks and unmarks orphan state with conditional transitions', async () => { + const store = createAwsDynamoDbRunnerStateStore(); + + await store.markOrphan('runner-123'); + await store.unmarkOrphan('runner-123'); + + const calls = mockDynamoDbClient.commandCalls(UpdateItemCommand); + expect(calls[0].args[0].input.ExpressionAttributeValues).toMatchObject({ + ':state': { S: 'orphan' }, + ':expectedState0': { S: 'active' }, + }); + expect(calls[0].args[0].input.UpdateExpression).toContain('REMOVE #expiresAt'); + expect(calls[0].args[0].input.ExpressionAttributeValues).not.toHaveProperty(':expiresAt'); + expect(calls[1].args[0].input.ExpressionAttributeValues).toMatchObject({ + ':state': { S: 'active' }, + ':expectedState0': { S: 'orphan' }, + }); + expect(calls[1].args[0].input.UpdateExpression).toContain('REMOVE #expiresAt'); + expect(calls[1].args[0].input.ExpressionAttributeValues).not.toHaveProperty(':expiresAt'); + }); + + it.each(['provisioning', 'active', 'orphan'] as const)( + 'claims termination and returns the prior %s state', + async (state) => { + mockDynamoDbClient.on(UpdateItemCommand).resolves({ Attributes: { state: { S: state } } }); + const store = createAwsDynamoDbRunnerStateStore(); + + await expect(store.beginTermination('runner-123')).resolves.toBe(state); + expect(mockDynamoDbClient).toHaveReceivedCommandWith(UpdateItemCommand, { + ReturnValues: 'ALL_OLD', + ConditionExpression: + 'attribute_exists(#scope) AND attribute_exists(#id) AND (#state IN (:provisioning, :active, :orphan) OR (#state = :terminating AND #updatedAt < :staleBefore))', + ExpressionAttributeValues: expect.objectContaining({ + ':terminating': { S: 'terminating' }, + ':expiresAt': { N: '1735776000' }, + ':provisioning': { S: 'provisioning' }, + ':active': { S: 'active' }, + ':orphan': { S: 'orphan' }, + ':staleBefore': { S: '2024-12-31T23:44:00.000Z' }, + }), + }); + }, + ); + + it('reclaims a stale terminating record on a later invocation without allowing an immediate double claim', async () => { + mockDynamoDbClient + .on(UpdateItemCommand) + .resolvesOnce({ Attributes: { state: { S: 'active' } } }) + .rejectsOnce(new ConditionalCheckFailedException({ $metadata: {}, message: 'lease is still held' })) + .resolvesOnce({ Attributes: { state: { S: 'terminating' } } }); + const store = createAwsDynamoDbRunnerStateStore(); + + await expect(store.beginTermination('runner-123')).resolves.toBe('active'); + await expect(store.beginTermination('runner-123')).resolves.toBeUndefined(); + vi.advanceTimersByTime(17 * 60 * 1000); + await expect(store.beginTermination('runner-123')).resolves.toBe('terminating'); + + const reclaimed = mockDynamoDbClient.commandCalls(UpdateItemCommand)[2].args[0].input; + expect(reclaimed.ExpressionAttributeValues?.[':staleBefore']).toEqual({ + S: '2025-01-01T00:01:00.000Z', + }); + }); + + it('returns undefined when another invocation already owns termination', async () => { + mockDynamoDbClient + .on(UpdateItemCommand) + .rejects(new ConditionalCheckFailedException({ $metadata: {}, message: 'condition failed' })); + const store = createAwsDynamoDbRunnerStateStore(); + + await expect(store.beginTermination('runner-123')).resolves.toBeUndefined(); + }); + + it('restores the safety TTL when cancellation returns a runner to provisioning', async () => { + const store = createAwsDynamoDbRunnerStateStore(); + + await store.cancelTermination('runner-123', 'provisioning'); + + expect(mockDynamoDbClient).toHaveReceivedCommandWith(UpdateItemCommand, { + UpdateExpression: 'SET #state = :state, #updatedAt = :updatedAt, #expiresAt = :expiresAt', + ExpressionAttributeValues: expect.objectContaining({ + ':state': { S: 'provisioning' }, + ':expiresAt': { N: '1735776000' }, + ':expectedState0': { S: 'terminating' }, + }), + }); + }); + + it.each(['active', 'orphan'] as const)('removes the safety TTL when cancellation restores %s', async (state) => { + const store = createAwsDynamoDbRunnerStateStore(); + + await store.cancelTermination('runner-123', state); + + expect(mockDynamoDbClient).toHaveReceivedCommandWith(UpdateItemCommand, { + UpdateExpression: 'SET #state = :state, #updatedAt = :updatedAt REMOVE #expiresAt', + ExpressionAttributeValues: expect.objectContaining({ + ':state': { S: state }, + ':expectedState0': { S: 'terminating' }, + }), + }); + const values = mockDynamoDbClient.commandCalls(UpdateItemCommand)[0].args[0].input.ExpressionAttributeValues; + expect(values).not.toHaveProperty(':expiresAt'); + }); + + it('deletes only a record whose termination was claimed', async () => { + const store = createAwsDynamoDbRunnerStateStore(); + + await store.delete('runner-123'); + + expect(mockDynamoDbClient).toHaveReceivedCommandWith(DeleteItemCommand, { + TableName: 'runner-state', + Key: { scope: { S: 'entry#linux-x64#runner-state' }, id: { S: 'runner#runner-123' } }, + ConditionExpression: '#state = :terminating', + ExpressionAttributeNames: { '#state': 'state' }, + ExpressionAttributeValues: { ':terminating': { S: 'terminating' } }, + }); + }); + + it.each([ + 'RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TABLE_NAME', + 'RUNNER_CONFIG_DYNAMODB_ENTRY_ID', + 'RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TTL_SECONDS', + ] as const)('rejects missing provider environment %s', (name) => { + delete process.env[name]; + + expect(() => createAwsDynamoDbRunnerStateStore()).toThrow(`Environment variable ${name} is not set`); + expect(mockDynamoDbClient.calls()).toHaveLength(0); + }); + + it.each(['0', '-1', '1.5', 'not-a-number'])('rejects invalid state TTL %j', (ttl) => { + process.env.RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TTL_SECONDS = ttl; + + expect(() => createAwsDynamoDbRunnerStateStore()).toThrow( + 'Environment variable RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TTL_SECONDS must be a positive integer', + ); + }); + + it('validates records before writing', async () => { + const store = createAwsDynamoDbRunnerStateStore(); + + await expect(store.create({ ...createRecord(), computeProvider: ' ' })).rejects.toThrow( + "Runner state field 'computeProvider' must be a non-empty string", + ); + await expect(store.create({ ...createRecord(), runnerType: 'Team' as never })).rejects.toThrow( + "Runner state field 'runnerType' must be 'Org' or 'Repo'", + ); + await expect(store.activate('runner-123', { runnerLabels: [''] })).rejects.toThrow( + "Runner state field 'runnerLabels' must be a non-empty string", + ); + await expect(store.recordGitHubIdentity('runner-123', { githubRunnerId: '' })).rejects.toThrow( + "Runner state field 'githubRunnerId' must be a non-empty string", + ); + await expect(store.list({ computeProvider: '' })).rejects.toThrow( + "Runner state field 'computeProvider' must be a non-empty string", + ); + expect(mockDynamoDbClient.calls()).toHaveLength(0); + }); + + it('propagates non-conditional lifecycle errors', async () => { + const error = new Error('service unavailable'); + mockDynamoDbClient.on(UpdateItemCommand).rejects(error); + const store = createAwsDynamoDbRunnerStateStore(); + + await expect(store.beginTermination('runner-123')).rejects.toBe(error); + }); +}); + +function createRecord(): CreateRunnerStateRecord { + return { + runnerId: 'runner-123', + computeProvider: 'aws_ec2', + computeResourceId: 'i-123', + runnerName: 'ghr-runner-123', + runnerLabels: ['linux', 'x64'], + runnerOwner: 'github-aws-runners', + runnerType: 'Org', + metadata: [{ key: 'Environment', value: 'test' }], + }; +} + +function storedRecord(options: { runnerId?: string; resourceId?: string } = {}): Record { + const runnerId = options.runnerId ?? 'runner-123'; + return { + scope: { S: 'entry#linux-x64#runner-state' }, + id: { S: `runner#${runnerId}` }, + runner_id: { S: runnerId }, + compute_provider: { S: 'aws_ec2' }, + compute_resource_id: { S: options.resourceId ?? 'i-123' }, + runner_name: { S: `ghr-${runnerId}` }, + runner_labels: { L: [{ S: 'linux' }, { S: 'x64' }] }, + github_runner_id: { S: '9876' }, + runner_owner: { S: 'github-aws-runners' }, + runner_type: { S: 'Org' }, + state: { S: 'active' }, + created_at: { S: '2025-01-01T00:00:00.000Z' }, + updated_at: { S: '2025-01-01T00:01:00.000Z' }, + metadata: { L: [{ M: { key: { S: 'Environment' }, value: { S: 'test' } } }] }, + }; +} + +function expectedRecord(options: { runnerId?: string; resourceId?: string } = {}) { + const runnerId = options.runnerId ?? 'runner-123'; + return { + runnerId, + computeProvider: 'aws_ec2', + computeResourceId: options.resourceId ?? 'i-123', + runnerName: `ghr-${runnerId}`, + runnerLabels: ['linux', 'x64'], + githubRunnerId: '9876', + runnerOwner: 'github-aws-runners', + runnerType: 'Org', + state: 'active', + createdAt: '2025-01-01T00:00:00.000Z', + updatedAt: '2025-01-01T00:01:00.000Z', + metadata: [{ key: 'Environment', value: 'test' }], + }; +} diff --git a/lambdas/libs/storage-providers/aws/dynamodb/runner-state-store.ts b/lambdas/libs/storage-providers/aws/dynamodb/runner-state-store.ts new file mode 100644 index 0000000000..7263aa93c1 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/runner-state-store.ts @@ -0,0 +1,556 @@ +import { + ConditionalCheckFailedException, + DeleteItemCommand, + PutItemCommand, + QueryCommand, + UpdateItemCommand, + type AttributeValue, +} from '@aws-sdk/client-dynamodb'; + +import type { + CreateRunnerStateRecord, + RunnerConfigMetadata, + RunnerGitHubIdentity, + RunnerLifecycleState, + RunnerStateActivation, + RunnerStateFilter, + RunnerStateRecord, + RunnerStateStore, + RunnerType, +} from '../../core'; +import { getDynamoDbClient } from './client'; +import { positiveIntegerEnvironmentValue, requiredEnvironmentValue } from './environment'; +import { EXPIRES_AT_ATTRIBUTE, ID_ATTRIBUTE, runnerStateId, runnerStateScope, SCOPE_ATTRIBUTE } from './keys'; + +const RUNNER_ID_PREFIX = 'runner#'; +const RUNNER_ID_ATTRIBUTE = 'runner_id'; +const COMPUTE_PROVIDER_ATTRIBUTE = 'compute_provider'; +const COMPUTE_RESOURCE_ID_ATTRIBUTE = 'compute_resource_id'; +const RUNNER_NAME_ATTRIBUTE = 'runner_name'; +const RUNNER_LABELS_ATTRIBUTE = 'runner_labels'; +const GITHUB_RUNNER_ID_ATTRIBUTE = 'github_runner_id'; +const RUNNER_OWNER_ATTRIBUTE = 'runner_owner'; +const RUNNER_TYPE_ATTRIBUTE = 'runner_type'; +const STATE_ATTRIBUTE = 'state'; +const CREATED_AT_ATTRIBUTE = 'created_at'; +const UPDATED_AT_ATTRIBUTE = 'updated_at'; +const METADATA_ATTRIBUTE = 'metadata'; +// AWS Lambda can run for at most 15 minutes. One extra minute prevents a second +// invocation from reclaiming a termination while the original can still be running. +const TERMINATION_CLAIM_LEASE_MILLISECONDS = 16 * 60 * 1000; + +interface AwsDynamoDbRunnerStateStoreConfig { + tableName: string; + scope: string; + ttlSeconds: number; +} + +export function createAwsDynamoDbRunnerStateStore(): RunnerStateStore { + return new AwsDynamoDbRunnerStateStore({ + tableName: requiredEnvironmentValue('RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TABLE_NAME'), + scope: runnerStateScope(requiredEnvironmentValue('RUNNER_CONFIG_DYNAMODB_ENTRY_ID')), + ttlSeconds: positiveIntegerEnvironmentValue('RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TTL_SECONDS'), + }); +} + +class AwsDynamoDbRunnerStateStore implements RunnerStateStore { + constructor(private readonly config: AwsDynamoDbRunnerStateStoreConfig) {} + + async create(record: CreateRunnerStateRecord): Promise { + validateCreateRecord(record); + const now = new Date(); + const timestamp = now.toISOString(); + const item: Record = { + [SCOPE_ATTRIBUTE]: { S: this.config.scope }, + [ID_ATTRIBUTE]: { S: runnerStateId(record.runnerId) }, + [RUNNER_ID_ATTRIBUTE]: { S: record.runnerId }, + [COMPUTE_PROVIDER_ATTRIBUTE]: { S: record.computeProvider }, + [COMPUTE_RESOURCE_ID_ATTRIBUTE]: { S: record.computeResourceId }, + [RUNNER_OWNER_ATTRIBUTE]: { S: record.runnerOwner }, + [RUNNER_TYPE_ATTRIBUTE]: { S: record.runnerType }, + [STATE_ATTRIBUTE]: { S: 'provisioning' }, + [CREATED_AT_ATTRIBUTE]: { S: timestamp }, + [UPDATED_AT_ATTRIBUTE]: { S: timestamp }, + [EXPIRES_AT_ATTRIBUTE]: { N: expiresAt(now, this.config.ttlSeconds) }, + }; + + setOptionalString(item, RUNNER_NAME_ATTRIBUTE, record.runnerName); + setOptionalStringList(item, RUNNER_LABELS_ATTRIBUTE, record.runnerLabels); + setMetadata(item, record.metadata); + + await getDynamoDbClient().send( + new PutItemCommand({ + TableName: this.config.tableName, + Item: item, + ConditionExpression: 'attribute_not_exists(#scope) AND attribute_not_exists(#id)', + ExpressionAttributeNames: { + '#scope': SCOPE_ATTRIBUTE, + '#id': ID_ATTRIBUTE, + }, + }), + ); + } + + async activate(runnerId: string, activation: RunnerStateActivation = {}): Promise { + validateNonEmptyString(runnerId, 'runnerId'); + validateActivation(activation); + await this.transition(runnerId, ['provisioning'], 'active', activation); + } + + async recordGitHubIdentity(runnerId: string, identity: RunnerGitHubIdentity): Promise { + validateNonEmptyString(identity.githubRunnerId, 'githubRunnerId'); + validateActivation(identity); + await this.transition(runnerId, ['provisioning'], 'provisioning', identity); + } + + async list(filter: RunnerStateFilter = {}): Promise { + if (filter.computeProvider !== undefined) { + validateNonEmptyString(filter.computeProvider, 'computeProvider'); + } + + const records: RunnerStateRecord[] = []; + let exclusiveStartKey: Record | undefined; + + do { + const expressionAttributeNames: Record = { + '#scope': SCOPE_ATTRIBUTE, + '#id': ID_ATTRIBUTE, + }; + const expressionAttributeValues: Record = { + ':scope': { S: this.config.scope }, + ':runner': { S: RUNNER_ID_PREFIX }, + }; + + if (filter.computeProvider !== undefined) { + expressionAttributeNames['#computeProvider'] = COMPUTE_PROVIDER_ATTRIBUTE; + expressionAttributeValues[':computeProvider'] = { S: filter.computeProvider }; + } + + const result = await getDynamoDbClient().send( + new QueryCommand({ + TableName: this.config.tableName, + KeyConditionExpression: '#scope = :scope AND begins_with(#id, :runner)', + FilterExpression: filter.computeProvider === undefined ? undefined : '#computeProvider = :computeProvider', + ExpressionAttributeNames: expressionAttributeNames, + ExpressionAttributeValues: expressionAttributeValues, + ConsistentRead: true, + ExclusiveStartKey: exclusiveStartKey, + }), + ); + + for (const item of result.Items ?? []) { + records.push(parseRunnerStateRecord(item, this.config.scope)); + } + exclusiveStartKey = result.LastEvaluatedKey; + } while (exclusiveStartKey !== undefined); + + return records; + } + + async markOrphan(runnerId: string): Promise { + await this.transition(runnerId, ['active'], 'orphan'); + } + + async unmarkOrphan(runnerId: string): Promise { + await this.transition(runnerId, ['orphan'], 'active'); + } + + async beginTermination(runnerId: string): Promise { + validateNonEmptyString(runnerId, 'runnerId'); + const now = new Date(); + const staleBefore = new Date(now.getTime() - TERMINATION_CLAIM_LEASE_MILLISECONDS).toISOString(); + try { + const previous = ( + await getDynamoDbClient().send( + new UpdateItemCommand({ + TableName: this.config.tableName, + Key: this.key(runnerId), + UpdateExpression: 'SET #state = :terminating, #updatedAt = :updatedAt, #expiresAt = :expiresAt', + ConditionExpression: + 'attribute_exists(#scope) AND attribute_exists(#id) AND (#state IN (:provisioning, :active, :orphan) OR (#state = :terminating AND #updatedAt < :staleBefore))', + ExpressionAttributeNames: { + '#scope': SCOPE_ATTRIBUTE, + '#id': ID_ATTRIBUTE, + '#state': STATE_ATTRIBUTE, + '#updatedAt': UPDATED_AT_ATTRIBUTE, + '#expiresAt': EXPIRES_AT_ATTRIBUTE, + }, + ExpressionAttributeValues: { + ':provisioning': { S: 'provisioning' }, + ':active': { S: 'active' }, + ':orphan': { S: 'orphan' }, + ':terminating': { S: 'terminating' }, + ':updatedAt': { S: now.toISOString() }, + ':staleBefore': { S: staleBefore }, + ':expiresAt': { N: expiresAt(now, this.config.ttlSeconds) }, + }, + ReturnValues: 'ALL_OLD', + }), + ) + ).Attributes; + const previousState = previous?.[STATE_ATTRIBUTE]?.S; + if (previousState === undefined || !isRunnerLifecycleState(previousState)) { + throw new Error(`Runner state item '${this.config.scope}/${runnerStateId(runnerId)}' returned no prior state`); + } + return previousState; + } catch (error) { + if (error instanceof ConditionalCheckFailedException) { + return undefined; + } + throw error; + } + } + + async cancelTermination(runnerId: string, restoreState: 'provisioning' | 'active' | 'orphan'): Promise { + if (restoreState !== 'provisioning' && restoreState !== 'active' && restoreState !== 'orphan') { + throw new Error("Runner state field 'restoreState' must be 'provisioning', 'active', or 'orphan'"); + } + await this.transition(runnerId, ['terminating'], restoreState); + } + + async delete(runnerId: string): Promise { + validateNonEmptyString(runnerId, 'runnerId'); + await getDynamoDbClient().send( + new DeleteItemCommand({ + TableName: this.config.tableName, + Key: this.key(runnerId), + ConditionExpression: '#state = :terminating', + ExpressionAttributeNames: { + '#state': STATE_ATTRIBUTE, + }, + ExpressionAttributeValues: { + ':terminating': { S: 'terminating' }, + }, + }), + ); + } + + private async transition( + runnerId: string, + expectedStates: RunnerLifecycleState[], + state: RunnerLifecycleState, + activation: RunnerStateActivation = {}, + returnOldState = false, + ): Promise | undefined> { + validateNonEmptyString(runnerId, 'runnerId'); + const now = new Date(); + const expressionAttributeNames: Record = { + '#scope': SCOPE_ATTRIBUTE, + '#id': ID_ATTRIBUTE, + '#state': STATE_ATTRIBUTE, + '#updatedAt': UPDATED_AT_ATTRIBUTE, + '#expiresAt': EXPIRES_AT_ATTRIBUTE, + }; + const expressionAttributeValues: Record = { + ':state': { S: state }, + ':updatedAt': { S: now.toISOString() }, + }; + const updates = ['#state = :state', '#updatedAt = :updatedAt']; + const hasSafetyTtl = state === 'provisioning' || state === 'terminating'; + if (hasSafetyTtl) { + expressionAttributeValues[':expiresAt'] = { N: expiresAt(now, this.config.ttlSeconds) }; + updates.push('#expiresAt = :expiresAt'); + } + addActivationUpdates(updates, expressionAttributeNames, expressionAttributeValues, activation); + + const expectedStateValues = expectedStates.map((expectedState, index) => { + const placeholder = `:expectedState${index}`; + expressionAttributeValues[placeholder] = { S: expectedState }; + return placeholder; + }); + + const result = await getDynamoDbClient().send( + new UpdateItemCommand({ + TableName: this.config.tableName, + Key: this.key(runnerId), + UpdateExpression: `SET ${updates.join(', ')}${hasSafetyTtl ? '' : ' REMOVE #expiresAt'}`, + ConditionExpression: `attribute_exists(#scope) AND attribute_exists(#id) AND #state IN (${expectedStateValues.join(', ')})`, + ExpressionAttributeNames: expressionAttributeNames, + ExpressionAttributeValues: expressionAttributeValues, + ReturnValues: returnOldState ? 'ALL_OLD' : undefined, + }), + ); + return result.Attributes; + } + + private key(runnerId: string): Record { + return { + [SCOPE_ATTRIBUTE]: { S: this.config.scope }, + [ID_ATTRIBUTE]: { S: runnerStateId(runnerId) }, + }; + } +} + +function parseRunnerStateRecord(item: Record, scope: string): RunnerStateRecord { + const id = requiredStringAttribute(item, ID_ATTRIBUTE, scope); + const runnerType = requiredStringAttribute(item, RUNNER_TYPE_ATTRIBUTE, `${scope}/${id}`); + if (runnerType !== 'Org' && runnerType !== 'Repo') { + throw invalidItem(`${scope}/${id}`, RUNNER_TYPE_ATTRIBUTE); + } + + return { + runnerId: requiredStringAttribute(item, RUNNER_ID_ATTRIBUTE, `${scope}/${id}`), + computeProvider: requiredStringAttribute(item, COMPUTE_PROVIDER_ATTRIBUTE, `${scope}/${id}`), + computeResourceId: requiredStringAttribute(item, COMPUTE_RESOURCE_ID_ATTRIBUTE, `${scope}/${id}`), + runnerName: optionalStringAttribute(item, RUNNER_NAME_ATTRIBUTE, `${scope}/${id}`), + runnerLabels: optionalStringListAttribute(item, RUNNER_LABELS_ATTRIBUTE, `${scope}/${id}`), + githubRunnerId: optionalStringAttribute(item, GITHUB_RUNNER_ID_ATTRIBUTE, `${scope}/${id}`), + runnerOwner: requiredStringAttribute(item, RUNNER_OWNER_ATTRIBUTE, `${scope}/${id}`), + runnerType, + state: requiredLifecycleState(item, `${scope}/${id}`), + createdAt: requiredTimestampAttribute(item, CREATED_AT_ATTRIBUTE, `${scope}/${id}`), + updatedAt: requiredTimestampAttribute(item, UPDATED_AT_ATTRIBUTE, `${scope}/${id}`), + metadata: optionalMetadataAttribute(item, `${scope}/${id}`), + }; +} + +function requiredLifecycleState(item: Record, itemId: string): RunnerLifecycleState { + const state = requiredStringAttribute(item, STATE_ATTRIBUTE, itemId); + if (!isRunnerLifecycleState(state)) { + throw invalidItem(itemId, STATE_ATTRIBUTE); + } + return state; +} + +function isRunnerLifecycleState(value: string): value is RunnerLifecycleState { + return value === 'provisioning' || value === 'active' || value === 'orphan' || value === 'terminating'; +} + +function requiredStringAttribute(item: Record, name: string, itemId: string): string { + const value = item[name]?.S; + if (value === undefined || value.trim() === '') { + throw invalidItem(itemId, name); + } + return value; +} + +function optionalStringAttribute( + item: Record, + name: string, + itemId: string, +): string | undefined { + if (item[name] === undefined) { + return undefined; + } + return requiredStringAttribute(item, name, itemId); +} + +function optionalStringListAttribute( + item: Record, + name: string, + itemId: string, +): string[] | undefined { + const attribute = item[name]; + if (attribute === undefined) { + return undefined; + } + if (!attribute.L) { + throw invalidItem(itemId, name); + } + + return attribute.L.map((value) => { + if (value.S === undefined || value.S.trim() === '') { + throw invalidItem(itemId, name); + } + return value.S; + }); +} + +function requiredTimestampAttribute(item: Record, name: string, itemId: string): string { + const value = requiredStringAttribute(item, name, itemId); + try { + if (new Date(value).toISOString() !== value) { + throw invalidItem(itemId, name); + } + } catch { + throw invalidItem(itemId, name); + } + return value; +} + +function optionalMetadataAttribute( + item: Record, + itemId: string, +): RunnerConfigMetadata[] | undefined { + const metadata = item[METADATA_ATTRIBUTE]; + if (metadata === undefined) { + return undefined; + } + if (!metadata.L) { + throw invalidItem(itemId, METADATA_ATTRIBUTE); + } + + return metadata.L.map((entry) => { + if (!entry.M) { + throw invalidItem(itemId, METADATA_ATTRIBUTE); + } + return { + key: requiredStringAttribute(entry.M, 'key', itemId), + value: requiredMetadataValue(entry.M, itemId), + }; + }); +} + +function validateCreateRecord(record: CreateRunnerStateRecord): void { + validateNonEmptyString(record.runnerId, 'runnerId'); + validateNonEmptyString(record.computeProvider, 'computeProvider'); + validateNonEmptyString(record.computeResourceId, 'computeResourceId'); + validateOptionalString(record.runnerName, 'runnerName'); + validateOptionalStringList(record.runnerLabels, 'runnerLabels'); + validateNonEmptyString(record.runnerOwner, 'runnerOwner'); + validateRunnerType(record.runnerType); + for (const metadata of record.metadata ?? []) { + validateNonEmptyString(metadata.key, 'metadata.key'); + validateString(metadata.value, 'metadata.value'); + } +} + +function validateActivation(activation: RunnerStateActivation): void { + validateOptionalString(activation.runnerName, 'runnerName'); + validateOptionalStringList(activation.runnerLabels, 'runnerLabels'); + validateOptionalString(activation.githubRunnerId, 'githubRunnerId'); + for (const metadata of activation.metadata ?? []) { + validateNonEmptyString(metadata.key, 'metadata.key'); + validateString(metadata.value, 'metadata.value'); + } +} + +function validateRunnerType(value: RunnerType): void { + if (value !== 'Org' && value !== 'Repo') { + throw new Error("Runner state field 'runnerType' must be 'Org' or 'Repo'"); + } +} + +function validateOptionalString(value: string | undefined, name: string): void { + if (value !== undefined) { + validateNonEmptyString(value, name); + } +} + +function validateOptionalStringList(values: string[] | undefined, name: string): void { + for (const value of values ?? []) { + validateNonEmptyString(value, name); + } +} + +function validateNonEmptyString(value: string, name: string): void { + if (typeof value !== 'string' || value.trim() === '') { + throw new Error(`Runner state field '${name}' must be a non-empty string`); + } +} + +function validateString(value: string, name: string): void { + if (typeof value !== 'string') { + throw new Error(`Runner state field '${name}' must be a string`); + } +} + +function requiredMetadataValue(item: Record, itemId: string): string { + const value = item.value?.S; + if (value === undefined) { + throw invalidItem(itemId, METADATA_ATTRIBUTE); + } + return value; +} + +function setOptionalString(item: Record, name: string, value: string | undefined): void { + if (value !== undefined) { + item[name] = { S: value }; + } +} + +function setOptionalStringList(item: Record, name: string, values: string[] | undefined): void { + if (values !== undefined) { + item[name] = { L: values.map((value) => ({ S: value })) }; + } +} + +function addActivationUpdates( + updates: string[], + names: Record, + values: Record, + activation: RunnerStateActivation, +): void { + addOptionalStringUpdate( + updates, + names, + values, + '#runnerName', + ':runnerName', + RUNNER_NAME_ATTRIBUTE, + activation.runnerName, + ); + addOptionalStringListUpdate( + updates, + names, + values, + '#runnerLabels', + ':runnerLabels', + RUNNER_LABELS_ATTRIBUTE, + activation.runnerLabels, + ); + addOptionalStringUpdate( + updates, + names, + values, + '#githubRunnerId', + ':githubRunnerId', + GITHUB_RUNNER_ID_ATTRIBUTE, + activation.githubRunnerId, + ); + if (activation.metadata !== undefined) { + names['#metadata'] = METADATA_ATTRIBUTE; + values[':metadata'] = { + L: activation.metadata.map(({ key, value }) => ({ M: { key: { S: key }, value: { S: value } } })), + }; + updates.push('#metadata = :metadata'); + } +} + +function addOptionalStringUpdate( + updates: string[], + names: Record, + values: Record, + namePlaceholder: string, + valuePlaceholder: string, + attributeName: string, + value: string | undefined, +): void { + if (value !== undefined) { + names[namePlaceholder] = attributeName; + values[valuePlaceholder] = { S: value }; + updates.push(`${namePlaceholder} = ${valuePlaceholder}`); + } +} + +function addOptionalStringListUpdate( + updates: string[], + names: Record, + values: Record, + namePlaceholder: string, + valuePlaceholder: string, + attributeName: string, + value: string[] | undefined, +): void { + if (value !== undefined) { + names[namePlaceholder] = attributeName; + values[valuePlaceholder] = { L: value.map((entry) => ({ S: entry })) }; + updates.push(`${namePlaceholder} = ${valuePlaceholder}`); + } +} + +function setMetadata(item: Record, metadata: RunnerConfigMetadata[] | undefined): void { + if (metadata && metadata.length > 0) { + item[METADATA_ATTRIBUTE] = { + L: metadata.map(({ key, value }) => ({ M: { key: { S: key }, value: { S: value } } })), + }; + } +} + +function expiresAt(now: Date, ttlSeconds: number): string { + return (Math.floor(now.getTime() / 1000) + ttlSeconds).toString(); +} + +function invalidItem(itemId: string, attribute: string): Error { + return new Error(`Runner state item '${itemId}' has an invalid '${attribute}' attribute`); +} diff --git a/lambdas/libs/storage-providers/core/index.ts b/lambdas/libs/storage-providers/core/index.ts index e6060eae2a..cf005b478c 100644 --- a/lambdas/libs/storage-providers/core/index.ts +++ b/lambdas/libs/storage-providers/core/index.ts @@ -9,6 +9,7 @@ export interface RunnerConfigMetadata { export interface RunnerConfigRecord { runnerId: string; value: string; + accessScope?: string; } export interface RunnerConfigStore { @@ -65,3 +66,53 @@ export interface RunnerGroupCacheStore { export interface RunnerMatcherConfigStore { get(): Promise; } + +export type RunnerType = 'Org' | 'Repo'; +export type RunnerLifecycleState = 'provisioning' | 'active' | 'orphan' | 'terminating'; + +export interface RunnerStateRecord { + runnerId: string; + computeProvider: string; + computeResourceId: string; + runnerName?: string; + runnerLabels?: string[]; + githubRunnerId?: string; + runnerOwner: string; + runnerType: RunnerType; + state: RunnerLifecycleState; + createdAt: string; + updatedAt: string; + metadata?: RunnerConfigMetadata[]; +} + +export type CreateRunnerStateRecord = Omit; + +export interface RunnerStateActivation { + runnerName?: string; + runnerLabels?: string[]; + githubRunnerId?: string; + metadata?: RunnerConfigMetadata[]; +} + +export type RunnerGitHubIdentity = RunnerStateActivation & { githubRunnerId: string }; + +export interface RunnerStateFilter { + computeProvider?: string; +} + +/** + * Provider-neutral index of compute resources that implement GitHub runners. + * Runner bootstrap configuration is deliberately stored by RunnerConfigStore + * in a separate item because it contains a short-lived secret payload. + */ +export interface RunnerStateStore { + create(record: CreateRunnerStateRecord): Promise; + recordGitHubIdentity(runnerId: string, identity: RunnerGitHubIdentity): Promise; + activate(runnerId: string, activation?: RunnerStateActivation): Promise; + list(filter?: RunnerStateFilter): Promise; + markOrphan(runnerId: string): Promise; + unmarkOrphan(runnerId: string): Promise; + beginTermination(runnerId: string): Promise; + cancelTermination(runnerId: string, restoreState: 'provisioning' | 'active' | 'orphan'): Promise; + delete(runnerId: string): Promise; +} diff --git a/lambdas/libs/storage-providers/github-app-credentials.test.ts b/lambdas/libs/storage-providers/github-app-credentials.test.ts new file mode 100644 index 0000000000..8b28f4ae8e --- /dev/null +++ b/lambdas/libs/storage-providers/github-app-credentials.test.ts @@ -0,0 +1,97 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAwsDynamoDbGitHubAppCredentialsStore } from './aws/dynamodb/github-app-credentials-store'; +import { createAwsSsmGitHubAppCredentialsStore } from './aws/ssm/github-app-credentials-store'; +import type { GitHubAppCredentialsStore } from './core'; +import { getGitHubAppCredentialsStore, resetGitHubAppCredentialsStore } from './github-app-credentials'; + +vi.mock('./aws/dynamodb/github-app-credentials-store', () => ({ + createAwsDynamoDbGitHubAppCredentialsStore: vi.fn(), +})); +vi.mock('./aws/ssm/github-app-credentials-store', () => ({ + createAwsSsmGitHubAppCredentialsStore: vi.fn(), +})); + +const createAwsDynamoDbStoreMock = vi.mocked(createAwsDynamoDbGitHubAppCredentialsStore); +const createAwsSsmStoreMock = vi.mocked(createAwsSsmGitHubAppCredentialsStore); +const cleanEnv = process.env; + +describe('GitHub App credentials store selection', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env = { ...cleanEnv }; + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + resetGitHubAppCredentialsStore(); + }); + + it.each([undefined, '', ' ', 'aws_ssm', ' AWS_SSM '])('uses aws_ssm for selector input %j', (provider) => { + setProvider(provider); + const store = stubSsmStore(); + + expect(getGitHubAppCredentialsStore()).toBe(store); + expect(createAwsSsmStoreMock).toHaveBeenCalledOnce(); + expect(createAwsDynamoDbStoreMock).not.toHaveBeenCalled(); + }); + + it.each(['aws_dynamodb', ' AWS_DYNAMODB '])('uses aws_dynamodb for selector input %j', (provider) => { + setProvider(provider); + const store = stubDynamoDbStore(); + + expect(getGitHubAppCredentialsStore()).toBe(store); + expect(createAwsDynamoDbStoreMock).toHaveBeenCalledOnce(); + expect(createAwsSsmStoreMock).not.toHaveBeenCalled(); + }); + + it('rejects an unsupported provider before creating a store', () => { + setProvider('not-registered'); + + expect(() => getGitHubAppCredentialsStore()).toThrow("Unsupported runner config storage provider 'not-registered'"); + expect(createAwsSsmStoreMock).not.toHaveBeenCalled(); + expect(createAwsDynamoDbStoreMock).not.toHaveBeenCalled(); + }); + + it('creates the selected store lazily and caches it', () => { + const store = stubSsmStore(); + + expect(createAwsSsmStoreMock).not.toHaveBeenCalled(); + const first = getGitHubAppCredentialsStore(); + setProvider('not-registered'); + const second = getGitHubAppCredentialsStore(); + + expect(first).toBe(store); + expect(second).toBe(store); + expect(createAwsSsmStoreMock).toHaveBeenCalledOnce(); + }); + + it('selects again after the test reset', () => { + const firstStore = stubSsmStore(); + expect(getGitHubAppCredentialsStore()).toBe(firstStore); + + const secondStore = { get: vi.fn() } satisfies GitHubAppCredentialsStore; + createAwsSsmStoreMock.mockReturnValue(secondStore); + resetGitHubAppCredentialsStore(); + + expect(getGitHubAppCredentialsStore()).toBe(secondStore); + expect(createAwsSsmStoreMock).toHaveBeenCalledTimes(2); + }); +}); + +function setProvider(provider: string | undefined): void { + if (provider === undefined) { + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + } else { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + } +} + +function stubSsmStore(): GitHubAppCredentialsStore { + const store = { get: vi.fn() } satisfies GitHubAppCredentialsStore; + createAwsSsmStoreMock.mockReturnValue(store); + return store; +} + +function stubDynamoDbStore(): GitHubAppCredentialsStore { + const store = { get: vi.fn() } satisfies GitHubAppCredentialsStore; + createAwsDynamoDbStoreMock.mockReturnValue(store); + return store; +} diff --git a/lambdas/libs/storage-providers/github-app-credentials.ts b/lambdas/libs/storage-providers/github-app-credentials.ts new file mode 100644 index 0000000000..11322d826d --- /dev/null +++ b/lambdas/libs/storage-providers/github-app-credentials.ts @@ -0,0 +1,25 @@ +import { createAwsDynamoDbGitHubAppCredentialsStore } from './aws/dynamodb/github-app-credentials-store'; +import { createAwsSsmGitHubAppCredentialsStore } from './aws/ssm/github-app-credentials-store'; +import type { GitHubAppCredentialsStore } from './core'; +import type {} from './environment'; +import { resolveRunnerConfigStorageProvider, type RunnerConfigStorageProvider } from './provider'; + +type GitHubAppCredentialsStoreFactory = () => GitHubAppCredentialsStore; + +const providerFactories = { + aws_ssm: createAwsSsmGitHubAppCredentialsStore, + aws_dynamodb: createAwsDynamoDbGitHubAppCredentialsStore, +} as const satisfies Record; + +let githubAppCredentialsStore: GitHubAppCredentialsStore | undefined; + +export function getGitHubAppCredentialsStore(): GitHubAppCredentialsStore { + githubAppCredentialsStore ??= + providerFactories[resolveRunnerConfigStorageProvider(process.env.RUNNER_CONFIG_STORAGE_PROVIDER)](); + return githubAppCredentialsStore; +} + +// Test-only reset for cases that need to exercise first-use environment selection. +export function resetGitHubAppCredentialsStore(): void { + githubAppCredentialsStore = undefined; +} diff --git a/lambdas/libs/storage-providers/github-webhook-secret.test.ts b/lambdas/libs/storage-providers/github-webhook-secret.test.ts index 5888e735b0..9b66f840be 100644 --- a/lambdas/libs/storage-providers/github-webhook-secret.test.ts +++ b/lambdas/libs/storage-providers/github-webhook-secret.test.ts @@ -1,14 +1,19 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { createAwsDynamoDbGitHubWebhookSecretStore } from './aws/dynamodb/github-webhook-secret-store'; import { createAwsSsmGitHubWebhookSecretStore } from './aws/ssm/github-webhook-secret-store'; import type { GitHubWebhookSecretStore } from './core'; import { getGitHubWebhookSecretStore, resetGitHubWebhookSecretStore } from './github-webhook-secret'; +vi.mock('./aws/dynamodb/github-webhook-secret-store', () => ({ + createAwsDynamoDbGitHubWebhookSecretStore: vi.fn(), +})); vi.mock('./aws/ssm/github-webhook-secret-store', () => ({ createAwsSsmGitHubWebhookSecretStore: vi.fn(), })); -const createAwsSsmGitHubWebhookSecretStoreMock = vi.mocked(createAwsSsmGitHubWebhookSecretStore); +const createAwsDynamoDbStoreMock = vi.mocked(createAwsDynamoDbGitHubWebhookSecretStore); +const createAwsSsmStoreMock = vi.mocked(createAwsSsmGitHubWebhookSecretStore); const cleanEnv = process.env; describe('GitHub webhook secret store selection', () => { @@ -19,52 +24,55 @@ describe('GitHub webhook secret store selection', () => { resetGitHubWebhookSecretStore(); }); - it.each([undefined, '', ' '])('uses aws_ssm for default selector input %j', (provider) => { + it.each([undefined, '', ' ', 'aws_ssm', ' AWS_SSM '])('uses aws_ssm for selector input %j', (provider) => { setProvider(provider); - const store = stubStore(); + const store = stubSsmStore(); expect(getGitHubWebhookSecretStore()).toBe(store); - expect(createAwsSsmGitHubWebhookSecretStoreMock).toHaveBeenCalledOnce(); + expect(createAwsSsmStoreMock).toHaveBeenCalledOnce(); + expect(createAwsDynamoDbStoreMock).not.toHaveBeenCalled(); }); - it.each(['aws_ssm', ' AWS_SSM '])('uses aws_ssm for explicit selector input %j', (provider) => { - process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; - const store = stubStore(); + it.each(['aws_dynamodb', ' AWS_DYNAMODB '])('uses aws_dynamodb for selector input %j', (provider) => { + setProvider(provider); + const store = stubDynamoDbStore(); expect(getGitHubWebhookSecretStore()).toBe(store); - expect(createAwsSsmGitHubWebhookSecretStoreMock).toHaveBeenCalledOnce(); + expect(createAwsDynamoDbStoreMock).toHaveBeenCalledOnce(); + expect(createAwsSsmStoreMock).not.toHaveBeenCalled(); }); - it('rejects an unsupported provider on first use', () => { - process.env.RUNNER_CONFIG_STORAGE_PROVIDER = 'not-registered'; + it('rejects an unsupported provider before creating a store', () => { + setProvider('not-registered'); expect(() => getGitHubWebhookSecretStore()).toThrow("Unsupported runner config storage provider 'not-registered'"); - expect(createAwsSsmGitHubWebhookSecretStoreMock).not.toHaveBeenCalled(); + expect(createAwsSsmStoreMock).not.toHaveBeenCalled(); + expect(createAwsDynamoDbStoreMock).not.toHaveBeenCalled(); }); - it('selects lazily and caches the created store', () => { - const store = stubStore(); + it('creates the selected store lazily and caches it', () => { + const store = stubSsmStore(); - expect(createAwsSsmGitHubWebhookSecretStoreMock).not.toHaveBeenCalled(); + expect(createAwsSsmStoreMock).not.toHaveBeenCalled(); const first = getGitHubWebhookSecretStore(); - process.env.RUNNER_CONFIG_STORAGE_PROVIDER = 'not-registered'; + setProvider('not-registered'); const second = getGitHubWebhookSecretStore(); expect(first).toBe(store); expect(second).toBe(store); - expect(createAwsSsmGitHubWebhookSecretStoreMock).toHaveBeenCalledOnce(); + expect(createAwsSsmStoreMock).toHaveBeenCalledOnce(); }); it('selects again after the test reset', () => { - const firstStore = stubStore(); + const firstStore = stubSsmStore(); expect(getGitHubWebhookSecretStore()).toBe(firstStore); const secondStore = { get: vi.fn() } satisfies GitHubWebhookSecretStore; - createAwsSsmGitHubWebhookSecretStoreMock.mockReturnValue(secondStore); + createAwsSsmStoreMock.mockReturnValue(secondStore); resetGitHubWebhookSecretStore(); expect(getGitHubWebhookSecretStore()).toBe(secondStore); - expect(createAwsSsmGitHubWebhookSecretStoreMock).toHaveBeenCalledTimes(2); + expect(createAwsSsmStoreMock).toHaveBeenCalledTimes(2); }); }); @@ -76,8 +84,14 @@ function setProvider(provider: string | undefined): void { } } -function stubStore(): GitHubWebhookSecretStore { +function stubSsmStore(): GitHubWebhookSecretStore { + const store = { get: vi.fn() } satisfies GitHubWebhookSecretStore; + createAwsSsmStoreMock.mockReturnValue(store); + return store; +} + +function stubDynamoDbStore(): GitHubWebhookSecretStore { const store = { get: vi.fn() } satisfies GitHubWebhookSecretStore; - createAwsSsmGitHubWebhookSecretStoreMock.mockReturnValue(store); + createAwsDynamoDbStoreMock.mockReturnValue(store); return store; } diff --git a/lambdas/libs/storage-providers/github-webhook-secret.ts b/lambdas/libs/storage-providers/github-webhook-secret.ts index f13df08718..8fd0a1758c 100644 --- a/lambdas/libs/storage-providers/github-webhook-secret.ts +++ b/lambdas/libs/storage-providers/github-webhook-secret.ts @@ -1,3 +1,4 @@ +import { createAwsDynamoDbGitHubWebhookSecretStore } from './aws/dynamodb/github-webhook-secret-store'; import { createAwsSsmGitHubWebhookSecretStore } from './aws/ssm/github-webhook-secret-store'; import type { GitHubWebhookSecretStore } from './core'; import type {} from './environment'; @@ -7,6 +8,7 @@ type GitHubWebhookSecretStoreFactory = () => GitHubWebhookSecretStore; const providerFactories = { aws_ssm: createAwsSsmGitHubWebhookSecretStore, + aws_dynamodb: createAwsDynamoDbGitHubWebhookSecretStore, } as const satisfies Record; let githubWebhookSecretStore: GitHubWebhookSecretStore | undefined; diff --git a/lambdas/libs/storage-providers/index.ts b/lambdas/libs/storage-providers/index.ts index fcd4bd9338..01d4c3ecf0 100644 --- a/lambdas/libs/storage-providers/index.ts +++ b/lambdas/libs/storage-providers/index.ts @@ -1,4 +1,5 @@ export type { + CreateRunnerStateRecord, GitHubAppCredential, GitHubAppCredentialsStore, GitHubWebhookSecretStore, @@ -10,7 +11,14 @@ export type { RunnerConfigStore, RunnerGroupCacheRecord, RunnerGroupCacheStore, + RunnerGitHubIdentity, RunnerMatcherConfigStore, + RunnerLifecycleState, + RunnerStateActivation, + RunnerStateFilter, + RunnerStateRecord, + RunnerStateStore, + RunnerType, } from './core'; export { createRunnerConfigHousekeeper } from './runner-config-housekeeper'; export { createRunnerConfigConsumer, type RunnerConfigConsumerConfig } from './runner-config-consumer'; @@ -22,5 +30,9 @@ export { export type { RunnerConfigStorageProvider } from './provider'; export { createCommonStorage, createStorageProviders } from './storage-providers'; export type { StorageProviders, RunnerConfigStorage, CommonStorage } from './core'; +export { getGitHubAppCredentialsStore, resetGitHubAppCredentialsStore } from './github-app-credentials'; +export { getRunnerConfigStore, resetRunnerConfigStore } from './runner-config'; +export { getRunnerGroupCacheStore, resetRunnerGroupCacheStore } from './runner-group-cache'; export { getGitHubWebhookSecretStore, resetGitHubWebhookSecretStore } from './github-webhook-secret'; export { getRunnerMatcherConfigStore, resetRunnerMatcherConfigStore } from './runner-matcher-config'; +export { getRunnerStateStore, resetRunnerStateStore } from './runner-state'; diff --git a/lambdas/libs/storage-providers/package.json b/lambdas/libs/storage-providers/package.json index 330d60a3f6..1a8687ef32 100644 --- a/lambdas/libs/storage-providers/package.json +++ b/lambdas/libs/storage-providers/package.json @@ -20,6 +20,7 @@ "dependencies": { "@aws-github-runner/aws-powertools-util": "*", "@aws-github-runner/aws-ssm-util": "*", + "@aws-sdk/client-dynamodb": "^3.1009.0", "@aws-sdk/client-ssm": "^3.1009.0" }, "devDependencies": { diff --git a/lambdas/libs/storage-providers/provider.ts b/lambdas/libs/storage-providers/provider.ts index ff9ecb73cc..9a36fb8f69 100644 --- a/lambdas/libs/storage-providers/provider.ts +++ b/lambdas/libs/storage-providers/provider.ts @@ -1,7 +1,11 @@ export const runnerConfigStorageProvider = { awsSsm: 'aws_ssm', + awsDynamodb: 'aws_dynamodb', } as const; -export const runnerConfigStorageProviders = [runnerConfigStorageProvider.awsSsm] as const; +export const runnerConfigStorageProviders = [ + runnerConfigStorageProvider.awsSsm, + runnerConfigStorageProvider.awsDynamodb, +] as const; export type RunnerConfigStorageProvider = (typeof runnerConfigStorageProviders)[number]; diff --git a/lambdas/libs/storage-providers/runner-config.test.ts b/lambdas/libs/storage-providers/runner-config.test.ts new file mode 100644 index 0000000000..0c85093fa0 --- /dev/null +++ b/lambdas/libs/storage-providers/runner-config.test.ts @@ -0,0 +1,108 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAwsDynamoDbRunnerConfigStore } from './aws/dynamodb/runner-config-store'; +import { createAwsSsmRunnerConfigStore } from './aws/ssm/runner-config-store'; +import type { RunnerConfigStore } from './core'; +import { getRunnerConfigStore, resetRunnerConfigStore } from './runner-config'; + +vi.mock('./aws/dynamodb/runner-config-store', () => ({ + createAwsDynamoDbRunnerConfigStore: vi.fn(), +})); +vi.mock('./aws/ssm/runner-config-store', () => ({ + createAwsSsmRunnerConfigStore: vi.fn(), +})); + +const createAwsDynamoDbRunnerConfigStoreMock = vi.mocked(createAwsDynamoDbRunnerConfigStore); +const createAwsSsmRunnerConfigStoreMock = vi.mocked(createAwsSsmRunnerConfigStore); +const cleanEnv = process.env; + +describe('runner config store selection', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env = { ...cleanEnv }; + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + resetRunnerConfigStore(); + }); + + it.each([undefined, '', ' '])('uses aws_ssm for default selector input %j', (provider) => { + setProvider(provider); + const store = stubStore(); + + expect(getRunnerConfigStore()).toBe(store); + expect(createAwsSsmRunnerConfigStoreMock).toHaveBeenCalledOnce(); + expect(createAwsDynamoDbRunnerConfigStoreMock).not.toHaveBeenCalled(); + }); + + it.each(['aws_ssm', ' AWS_SSM '])('uses aws_ssm for explicit selector input %j', (provider) => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + const store = stubStore(); + + expect(getRunnerConfigStore()).toBe(store); + expect(createAwsSsmRunnerConfigStoreMock).toHaveBeenCalledOnce(); + expect(createAwsDynamoDbRunnerConfigStoreMock).not.toHaveBeenCalled(); + }); + + it.each(['aws_dynamodb', ' AWS_DYNAMODB '])('uses aws_dynamodb for explicit selector input %j', (provider) => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + const store = stubDynamoDbStore(); + + expect(getRunnerConfigStore()).toBe(store); + expect(createAwsDynamoDbRunnerConfigStoreMock).toHaveBeenCalledOnce(); + expect(createAwsSsmRunnerConfigStoreMock).not.toHaveBeenCalled(); + }); + + it('rejects an unsupported provider on first use', () => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = 'not-registered'; + + expect(createAwsSsmRunnerConfigStoreMock).not.toHaveBeenCalled(); + expect(createAwsDynamoDbRunnerConfigStoreMock).not.toHaveBeenCalled(); + expect(() => getRunnerConfigStore()).toThrow("Unsupported runner config storage provider 'not-registered'"); + expect(createAwsSsmRunnerConfigStoreMock).not.toHaveBeenCalled(); + expect(createAwsDynamoDbRunnerConfigStoreMock).not.toHaveBeenCalled(); + }); + + it('selects lazily and caches the created store', () => { + const store = stubStore(); + + expect(createAwsSsmRunnerConfigStoreMock).not.toHaveBeenCalled(); + const first = getRunnerConfigStore(); + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = 'not-registered'; + const second = getRunnerConfigStore(); + + expect(first).toBe(store); + expect(second).toBe(store); + expect(createAwsSsmRunnerConfigStoreMock).toHaveBeenCalledOnce(); + }); + + it('selects again after the test reset', () => { + const firstStore = stubStore(); + expect(getRunnerConfigStore()).toBe(firstStore); + + const secondStore = { create: vi.fn(), houseKeeper: vi.fn() } satisfies RunnerConfigStore; + createAwsSsmRunnerConfigStoreMock.mockReturnValue(secondStore); + resetRunnerConfigStore(); + + expect(getRunnerConfigStore()).toBe(secondStore); + expect(createAwsSsmRunnerConfigStoreMock).toHaveBeenCalledTimes(2); + }); +}); + +function setProvider(provider: string | undefined): void { + if (provider === undefined) { + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + } else { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + } +} + +function stubStore(): RunnerConfigStore { + const store = { create: vi.fn(), houseKeeper: vi.fn() } satisfies RunnerConfigStore; + createAwsSsmRunnerConfigStoreMock.mockReturnValue(store); + return store; +} + +function stubDynamoDbStore(): RunnerConfigStore { + const store = { create: vi.fn(), houseKeeper: vi.fn() } satisfies RunnerConfigStore; + createAwsDynamoDbRunnerConfigStoreMock.mockReturnValue(store); + return store; +} diff --git a/lambdas/libs/storage-providers/runner-config.ts b/lambdas/libs/storage-providers/runner-config.ts new file mode 100644 index 0000000000..d4d3998d15 --- /dev/null +++ b/lambdas/libs/storage-providers/runner-config.ts @@ -0,0 +1,25 @@ +import { createAwsDynamoDbRunnerConfigStore } from './aws/dynamodb/runner-config-store'; +import { createAwsSsmRunnerConfigStore } from './aws/ssm/runner-config-store'; +import type { RunnerConfigStore } from './core'; +import type {} from './environment'; +import { resolveRunnerConfigStorageProvider, type RunnerConfigStorageProvider } from './provider'; + +type RunnerConfigStoreFactory = () => RunnerConfigStore; + +const providerFactories = { + aws_ssm: createAwsSsmRunnerConfigStore, + aws_dynamodb: createAwsDynamoDbRunnerConfigStore, +} as const satisfies Record; + +let runnerConfigStore: RunnerConfigStore | undefined; + +export function getRunnerConfigStore(): RunnerConfigStore { + runnerConfigStore ??= + providerFactories[resolveRunnerConfigStorageProvider(process.env.RUNNER_CONFIG_STORAGE_PROVIDER)](); + return runnerConfigStore; +} + +// Test-only reset for cases that need to exercise first-use environment selection. +export function resetRunnerConfigStore(): void { + runnerConfigStore = undefined; +} diff --git a/lambdas/libs/storage-providers/runner-group-cache.test.ts b/lambdas/libs/storage-providers/runner-group-cache.test.ts new file mode 100644 index 0000000000..029e33be8a --- /dev/null +++ b/lambdas/libs/storage-providers/runner-group-cache.test.ts @@ -0,0 +1,106 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAwsDynamoDbRunnerGroupCacheStore } from './aws/dynamodb/runner-group-cache-store'; +import { createAwsSsmRunnerGroupCacheStore } from './aws/ssm/runner-group-cache-store'; +import type { RunnerGroupCacheStore } from './core'; +import { getRunnerGroupCacheStore, resetRunnerGroupCacheStore } from './runner-group-cache'; + +vi.mock('./aws/dynamodb/runner-group-cache-store', () => ({ + createAwsDynamoDbRunnerGroupCacheStore: vi.fn(), +})); +vi.mock('./aws/ssm/runner-group-cache-store', () => ({ + createAwsSsmRunnerGroupCacheStore: vi.fn(), +})); + +const createAwsDynamoDbRunnerGroupCacheStoreMock = vi.mocked(createAwsDynamoDbRunnerGroupCacheStore); +const createAwsSsmRunnerGroupCacheStoreMock = vi.mocked(createAwsSsmRunnerGroupCacheStore); +const cleanEnv = process.env; + +describe('runner group cache store selection', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env = { ...cleanEnv }; + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + resetRunnerGroupCacheStore(); + }); + + it.each([undefined, '', ' '])('uses aws_ssm for default selector input %j', (provider) => { + setProvider(provider); + const store = stubStore(); + + expect(getRunnerGroupCacheStore()).toBe(store); + expect(createAwsSsmRunnerGroupCacheStoreMock).toHaveBeenCalledOnce(); + expect(createAwsDynamoDbRunnerGroupCacheStoreMock).not.toHaveBeenCalled(); + }); + + it.each(['aws_ssm', ' AWS_SSM '])('uses aws_ssm for explicit selector input %j', (provider) => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + const store = stubStore(); + + expect(getRunnerGroupCacheStore()).toBe(store); + expect(createAwsSsmRunnerGroupCacheStoreMock).toHaveBeenCalledOnce(); + expect(createAwsDynamoDbRunnerGroupCacheStoreMock).not.toHaveBeenCalled(); + }); + + it.each(['aws_dynamodb', ' AWS_DYNAMODB '])('uses aws_dynamodb for explicit selector input %j', (provider) => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + const store = stubDynamoDbStore(); + + expect(getRunnerGroupCacheStore()).toBe(store); + expect(createAwsDynamoDbRunnerGroupCacheStoreMock).toHaveBeenCalledOnce(); + expect(createAwsSsmRunnerGroupCacheStoreMock).not.toHaveBeenCalled(); + }); + + it('rejects an unsupported provider on first use', () => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = 'not-registered'; + + expect(() => getRunnerGroupCacheStore()).toThrow("Unsupported runner config storage provider 'not-registered'"); + expect(createAwsSsmRunnerGroupCacheStoreMock).not.toHaveBeenCalled(); + expect(createAwsDynamoDbRunnerGroupCacheStoreMock).not.toHaveBeenCalled(); + }); + + it('selects lazily and caches the created store', () => { + const store = stubStore(); + + expect(createAwsSsmRunnerGroupCacheStoreMock).not.toHaveBeenCalled(); + const first = getRunnerGroupCacheStore(); + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = 'not-registered'; + const second = getRunnerGroupCacheStore(); + + expect(first).toBe(store); + expect(second).toBe(store); + expect(createAwsSsmRunnerGroupCacheStoreMock).toHaveBeenCalledOnce(); + }); + + it('selects again after the test reset', () => { + const firstStore = stubStore(); + expect(getRunnerGroupCacheStore()).toBe(firstStore); + + const secondStore = { get: vi.fn(), create: vi.fn() } satisfies RunnerGroupCacheStore; + createAwsSsmRunnerGroupCacheStoreMock.mockReturnValue(secondStore); + resetRunnerGroupCacheStore(); + + expect(getRunnerGroupCacheStore()).toBe(secondStore); + expect(createAwsSsmRunnerGroupCacheStoreMock).toHaveBeenCalledTimes(2); + }); +}); + +function setProvider(provider: string | undefined): void { + if (provider === undefined) { + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + } else { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + } +} + +function stubStore(): RunnerGroupCacheStore { + const store = { get: vi.fn(), create: vi.fn() } satisfies RunnerGroupCacheStore; + createAwsSsmRunnerGroupCacheStoreMock.mockReturnValue(store); + return store; +} + +function stubDynamoDbStore(): RunnerGroupCacheStore { + const store = { get: vi.fn(), create: vi.fn() } satisfies RunnerGroupCacheStore; + createAwsDynamoDbRunnerGroupCacheStoreMock.mockReturnValue(store); + return store; +} diff --git a/lambdas/libs/storage-providers/runner-group-cache.ts b/lambdas/libs/storage-providers/runner-group-cache.ts new file mode 100644 index 0000000000..3a91437755 --- /dev/null +++ b/lambdas/libs/storage-providers/runner-group-cache.ts @@ -0,0 +1,25 @@ +import { createAwsDynamoDbRunnerGroupCacheStore } from './aws/dynamodb/runner-group-cache-store'; +import { createAwsSsmRunnerGroupCacheStore } from './aws/ssm/runner-group-cache-store'; +import type { RunnerGroupCacheStore } from './core'; +import type {} from './environment'; +import { resolveRunnerConfigStorageProvider, type RunnerConfigStorageProvider } from './provider'; + +type RunnerGroupCacheStoreFactory = () => RunnerGroupCacheStore; + +const providerFactories = { + aws_ssm: createAwsSsmRunnerGroupCacheStore, + aws_dynamodb: createAwsDynamoDbRunnerGroupCacheStore, +} as const satisfies Record; + +let runnerGroupCacheStore: RunnerGroupCacheStore | undefined; + +export function getRunnerGroupCacheStore(): RunnerGroupCacheStore { + runnerGroupCacheStore ??= + providerFactories[resolveRunnerConfigStorageProvider(process.env.RUNNER_CONFIG_STORAGE_PROVIDER)](); + return runnerGroupCacheStore; +} + +// Test-only reset for cases that need to exercise first-use environment selection. +export function resetRunnerGroupCacheStore(): void { + runnerGroupCacheStore = undefined; +} diff --git a/lambdas/libs/storage-providers/runner-matcher-config.test.ts b/lambdas/libs/storage-providers/runner-matcher-config.test.ts index 0dd7f42ad3..981ff28567 100644 --- a/lambdas/libs/storage-providers/runner-matcher-config.test.ts +++ b/lambdas/libs/storage-providers/runner-matcher-config.test.ts @@ -1,13 +1,18 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { createAwsDynamoDbRunnerMatcherConfigStore } from './aws/dynamodb/runner-matcher-config-store'; import { createAwsSsmRunnerMatcherConfigStore } from './aws/ssm/runner-matcher-config-store'; import type { RunnerMatcherConfigStore } from './core'; import { getRunnerMatcherConfigStore, resetRunnerMatcherConfigStore } from './runner-matcher-config'; +vi.mock('./aws/dynamodb/runner-matcher-config-store', () => ({ + createAwsDynamoDbRunnerMatcherConfigStore: vi.fn(), +})); vi.mock('./aws/ssm/runner-matcher-config-store', () => ({ createAwsSsmRunnerMatcherConfigStore: vi.fn(), })); +const createAwsDynamoDbRunnerMatcherConfigStoreMock = vi.mocked(createAwsDynamoDbRunnerMatcherConfigStore); const createAwsSsmRunnerMatcherConfigStoreMock = vi.mocked(createAwsSsmRunnerMatcherConfigStore); const cleanEnv = process.env; @@ -25,6 +30,7 @@ describe('runner matcher config store selection', () => { expect(getRunnerMatcherConfigStore()).toBe(store); expect(createAwsSsmRunnerMatcherConfigStoreMock).toHaveBeenCalledOnce(); + expect(createAwsDynamoDbRunnerMatcherConfigStoreMock).not.toHaveBeenCalled(); }); it.each(['aws_ssm', ' AWS_SSM '])('uses aws_ssm for explicit selector input %j', (provider) => { @@ -33,6 +39,16 @@ describe('runner matcher config store selection', () => { expect(getRunnerMatcherConfigStore()).toBe(store); expect(createAwsSsmRunnerMatcherConfigStoreMock).toHaveBeenCalledOnce(); + expect(createAwsDynamoDbRunnerMatcherConfigStoreMock).not.toHaveBeenCalled(); + }); + + it.each(['aws_dynamodb', ' AWS_DYNAMODB '])('uses aws_dynamodb for explicit selector input %j', (provider) => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + const store = stubDynamoDbStore(); + + expect(getRunnerMatcherConfigStore()).toBe(store); + expect(createAwsDynamoDbRunnerMatcherConfigStoreMock).toHaveBeenCalledOnce(); + expect(createAwsSsmRunnerMatcherConfigStoreMock).not.toHaveBeenCalled(); }); it('rejects an unsupported provider on first use', () => { @@ -40,6 +56,7 @@ describe('runner matcher config store selection', () => { expect(() => getRunnerMatcherConfigStore()).toThrow("Unsupported runner config storage provider 'not-registered'"); expect(createAwsSsmRunnerMatcherConfigStoreMock).not.toHaveBeenCalled(); + expect(createAwsDynamoDbRunnerMatcherConfigStoreMock).not.toHaveBeenCalled(); }); it('selects lazily and caches the created store', () => { @@ -81,3 +98,9 @@ function stubStore(): RunnerMatcherConfigStore { createAwsSsmRunnerMatcherConfigStoreMock.mockReturnValue(store); return store; } + +function stubDynamoDbStore(): RunnerMatcherConfigStore { + const store = { get: vi.fn() } satisfies RunnerMatcherConfigStore; + createAwsDynamoDbRunnerMatcherConfigStoreMock.mockReturnValue(store); + return store; +} diff --git a/lambdas/libs/storage-providers/runner-matcher-config.ts b/lambdas/libs/storage-providers/runner-matcher-config.ts index 6d56d49754..c121fc2aaf 100644 --- a/lambdas/libs/storage-providers/runner-matcher-config.ts +++ b/lambdas/libs/storage-providers/runner-matcher-config.ts @@ -1,3 +1,4 @@ +import { createAwsDynamoDbRunnerMatcherConfigStore } from './aws/dynamodb/runner-matcher-config-store'; import { createAwsSsmRunnerMatcherConfigStore } from './aws/ssm/runner-matcher-config-store'; import type { RunnerMatcherConfigStore } from './core'; import type {} from './environment'; @@ -7,6 +8,7 @@ type RunnerMatcherConfigStoreFactory = () => RunnerMatcherConfigStore; const providerFactories = { aws_ssm: createAwsSsmRunnerMatcherConfigStore, + aws_dynamodb: createAwsDynamoDbRunnerMatcherConfigStore, } as const satisfies Record; let runnerMatcherConfigStore: RunnerMatcherConfigStore | undefined; diff --git a/lambdas/libs/storage-providers/runner-state.test.ts b/lambdas/libs/storage-providers/runner-state.test.ts new file mode 100644 index 0000000000..0d997614e7 --- /dev/null +++ b/lambdas/libs/storage-providers/runner-state.test.ts @@ -0,0 +1,89 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAwsDynamoDbRunnerStateStore } from './aws/dynamodb/runner-state-store'; +import type { RunnerStateStore } from './core'; +import { getRunnerStateStore, resetRunnerStateStore } from './runner-state'; + +vi.mock('./aws/dynamodb/runner-state-store', () => ({ + createAwsDynamoDbRunnerStateStore: vi.fn(), +})); + +const createAwsDynamoDbRunnerStateStoreMock = vi.mocked(createAwsDynamoDbRunnerStateStore); +const cleanEnv = process.env; + +describe('runner state store selection', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env = { ...cleanEnv }; + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + resetRunnerStateStore(); + }); + + it.each([undefined, '', ' ', 'aws_ssm', ' AWS_SSM '])( + 'returns no inventory capability for aws_ssm selector input %j', + (provider) => { + setProvider(provider); + + expect(getRunnerStateStore()).toBeUndefined(); + expect(createAwsDynamoDbRunnerStateStoreMock).not.toHaveBeenCalled(); + }, + ); + + it.each(['aws_dynamodb', ' AWS_DYNAMODB '])('creates the DynamoDB inventory for selector input %j', (provider) => { + setProvider(provider); + const store = stubStore(); + + expect(getRunnerStateStore()).toBe(store); + expect(createAwsDynamoDbRunnerStateStoreMock).toHaveBeenCalledOnce(); + }); + + it('rejects an unsupported provider before creating a store', () => { + setProvider('not-registered'); + + expect(() => getRunnerStateStore()).toThrow("Unsupported runner config storage provider 'not-registered'"); + expect(createAwsDynamoDbRunnerStateStoreMock).not.toHaveBeenCalled(); + }); + + it('caches the DynamoDB store until reset', () => { + setProvider('aws_dynamodb'); + const firstStore = stubStore(); + + expect(getRunnerStateStore()).toBe(firstStore); + expect(getRunnerStateStore()).toBe(firstStore); + expect(createAwsDynamoDbRunnerStateStoreMock).toHaveBeenCalledOnce(); + + const secondStore = createStubStore(); + createAwsDynamoDbRunnerStateStoreMock.mockReturnValue(secondStore); + resetRunnerStateStore(); + expect(getRunnerStateStore()).toBe(secondStore); + expect(createAwsDynamoDbRunnerStateStoreMock).toHaveBeenCalledTimes(2); + }); +}); + +function setProvider(provider: string | undefined): void { + if (provider === undefined) { + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + } else { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + } +} + +function stubStore(): RunnerStateStore { + const store = createStubStore(); + createAwsDynamoDbRunnerStateStoreMock.mockReturnValue(store); + return store; +} + +function createStubStore(): RunnerStateStore { + return { + create: vi.fn(), + recordGitHubIdentity: vi.fn(), + activate: vi.fn(), + list: vi.fn(), + markOrphan: vi.fn(), + unmarkOrphan: vi.fn(), + beginTermination: vi.fn(), + cancelTermination: vi.fn(), + delete: vi.fn(), + }; +} diff --git a/lambdas/libs/storage-providers/runner-state.ts b/lambdas/libs/storage-providers/runner-state.ts new file mode 100644 index 0000000000..e09a09d2b6 --- /dev/null +++ b/lambdas/libs/storage-providers/runner-state.ts @@ -0,0 +1,25 @@ +import { createAwsDynamoDbRunnerStateStore } from './aws/dynamodb/runner-state-store'; +import type { RunnerStateStore } from './core'; +import type {} from './environment'; +import { resolveRunnerConfigStorageProvider } from './provider'; + +let runnerStateStore: RunnerStateStore | undefined; + +export function getRunnerStateStore(): RunnerStateStore | undefined { + if (runnerStateStore) { + return runnerStateStore; + } + + const provider = resolveRunnerConfigStorageProvider(process.env.RUNNER_CONFIG_STORAGE_PROVIDER); + if (provider !== 'aws_dynamodb') { + return undefined; + } + + runnerStateStore = createAwsDynamoDbRunnerStateStore(); + return runnerStateStore; +} + +// Test-only reset for cases that need to exercise first-use environment selection. +export function resetRunnerStateStore(): void { + runnerStateStore = undefined; +} diff --git a/lambdas/libs/storage-providers/vitest.config.ts b/lambdas/libs/storage-providers/vitest.config.ts index a009dab1bf..300d39fbf6 100644 --- a/lambdas/libs/storage-providers/vitest.config.ts +++ b/lambdas/libs/storage-providers/vitest.config.ts @@ -15,6 +15,7 @@ export default mergeConfig(defaultConfig, { 'provider.ts', 'github-webhook-secret.ts', 'runner-matcher-config.ts', + 'runner-state.ts', 'core/**/*.ts', 'aws/**/*.ts', ], diff --git a/lambdas/yarn.lock b/lambdas/yarn.lock index 17e99e638d..2cbfaa1638 100644 --- a/lambdas/yarn.lock +++ b/lambdas/yarn.lock @@ -207,6 +207,7 @@ __metadata: dependencies: "@aws-github-runner/aws-powertools-util": "npm:*" "@aws-github-runner/aws-ssm-util": "npm:*" + "@aws-sdk/client-dynamodb": "npm:^3.1009.0" "@aws-sdk/client-ssm": "npm:^3.1009.0" aws-sdk-client-mock: "npm:^4.1.0" aws-sdk-client-mock-jest: "npm:^4.1.0" @@ -346,6 +347,24 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/client-dynamodb@npm:^3.1009.0": + version: 3.1124.0 + resolution: "@aws-sdk/client-dynamodb@npm:3.1124.0" + dependencies: + "@aws-sdk/core": "npm:^3.977.9" + "@aws-sdk/credential-provider-node": "npm:^3.972.82" + "@aws-sdk/dynamodb-codec": "npm:^3.973.44" + "@aws-sdk/middleware-endpoint-discovery": "npm:^3.972.30" + "@aws-sdk/types": "npm:^3.974.5" + "@smithy/core": "npm:^3.33.3" + "@smithy/fetch-http-handler": "npm:^5.7.2" + "@smithy/node-http-handler": "npm:^4.11.3" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/d0875ae63685617db38ea4033a5fd5ea229f548595561007c56baa6f50704abe435a222a5cd62eda89fdfa5a96fadaf1aa98e234d694ecb15281bfe92226d3b5 + languageName: node + linkType: hard + "@aws-sdk/client-ec2@npm:^3.1009.0": version: 3.1014.0 resolution: "@aws-sdk/client-ec2@npm:3.1014.0" @@ -624,6 +643,22 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/core@npm:^3.977.9": + version: 3.977.9 + resolution: "@aws-sdk/core@npm:3.977.9" + dependencies: + "@aws-sdk/types": "npm:^3.974.5" + "@aws-sdk/xml-builder": "npm:^3.972.40" + "@aws/lambda-invoke-store": "npm:^0.3.0" + "@smithy/core": "npm:^3.33.3" + "@smithy/signature-v4": "npm:^5.6.12" + "@smithy/types": "npm:^4.17.2" + bowser: "npm:^2.11.0" + tslib: "npm:^2.6.2" + checksum: 10c0/9118a8d05b7c27fe55fb5e793840193d1b311b6c0c2958e591bcada391a71783536ed9c8f4913cc9c4b7258dba650d625ed782669a12875348544c33f35aa8d6 + languageName: node + linkType: hard + "@aws-sdk/crc64-nvme@npm:^3.972.5": version: 3.972.5 resolution: "@aws-sdk/crc64-nvme@npm:3.972.5" @@ -647,6 +682,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-env@npm:^3.972.70": + version: 3.972.70 + resolution: "@aws-sdk/credential-provider-env@npm:3.972.70" + dependencies: + "@aws-sdk/core": "npm:^3.977.9" + "@aws-sdk/types": "npm:^3.974.5" + "@smithy/core": "npm:^3.33.3" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/f5d8f3f1021a83911f617dd6b36277486d0bdb174c107c4f7776a7e2c609a101aa03ada1cca9d97229ac79aea7797ec6392fe449029b87f7cd013360592bddf3 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-http@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-http@npm:3.972.23" @@ -665,6 +713,21 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-http@npm:^3.972.72": + version: 3.972.72 + resolution: "@aws-sdk/credential-provider-http@npm:3.972.72" + dependencies: + "@aws-sdk/core": "npm:^3.977.9" + "@aws-sdk/types": "npm:^3.974.5" + "@smithy/core": "npm:^3.33.3" + "@smithy/fetch-http-handler": "npm:^5.7.2" + "@smithy/node-http-handler": "npm:^4.11.3" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/aaa67dc42ce00713d92f931c620e36cb199533e6f1f892a6be76553986c86977442924e76f3743732a7b65e6f6777271789a6edbc70d5a86e0a9b6b5a3ef25f3 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-ini@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-ini@npm:3.972.23" @@ -687,6 +750,27 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-ini@npm:^3.973.15": + version: 3.973.15 + resolution: "@aws-sdk/credential-provider-ini@npm:3.973.15" + dependencies: + "@aws-sdk/core": "npm:^3.977.9" + "@aws-sdk/credential-provider-env": "npm:^3.972.70" + "@aws-sdk/credential-provider-http": "npm:^3.972.72" + "@aws-sdk/credential-provider-login": "npm:^3.972.77" + "@aws-sdk/credential-provider-process": "npm:^3.972.70" + "@aws-sdk/credential-provider-sso": "npm:^3.973.14" + "@aws-sdk/credential-provider-web-identity": "npm:^3.972.76" + "@aws-sdk/nested-clients": "npm:^3.997.44" + "@aws-sdk/types": "npm:^3.974.5" + "@smithy/core": "npm:^3.33.3" + "@smithy/credential-provider-imds": "npm:^4.4.16" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/1fe5f8c84cb9877a62084f9cbedebaed605ecc34891c44254d979ccd1282a5e164840d6dbb340773c311c028568f9ea642649bebf4b38cebb2d6d0039ad9de12 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-login@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-login@npm:3.972.23" @@ -703,6 +787,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-login@npm:^3.972.77": + version: 3.972.77 + resolution: "@aws-sdk/credential-provider-login@npm:3.972.77" + dependencies: + "@aws-sdk/core": "npm:^3.977.9" + "@aws-sdk/nested-clients": "npm:^3.997.44" + "@aws-sdk/types": "npm:^3.974.5" + "@smithy/core": "npm:^3.33.3" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/9be54fdf406325d462abdc0a2a182ba1ed39009fa18b60035705e6c3fae92dfe1afe5dba24a4fa3c8376f7cd2e1dad7a91152909f3b96dde0b215a88dfcd5330 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-node@npm:^3.972.24": version: 3.972.24 resolution: "@aws-sdk/credential-provider-node@npm:3.972.24" @@ -723,6 +821,25 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-node@npm:^3.972.82": + version: 3.972.82 + resolution: "@aws-sdk/credential-provider-node@npm:3.972.82" + dependencies: + "@aws-sdk/credential-provider-env": "npm:^3.972.70" + "@aws-sdk/credential-provider-http": "npm:^3.972.72" + "@aws-sdk/credential-provider-ini": "npm:^3.973.15" + "@aws-sdk/credential-provider-process": "npm:^3.972.70" + "@aws-sdk/credential-provider-sso": "npm:^3.973.14" + "@aws-sdk/credential-provider-web-identity": "npm:^3.972.76" + "@aws-sdk/types": "npm:^3.974.5" + "@smithy/core": "npm:^3.33.3" + "@smithy/credential-provider-imds": "npm:^4.4.16" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/c79c2381c1e76fecadb5adf9a8623946d819b20bfc0eaa236a384115027cdea3fe1f0a294c75edbb6aee1a0ccbbc4ec606c2eee8e1c4d720ed2ef281aa1175c8 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-process@npm:^3.972.21": version: 3.972.21 resolution: "@aws-sdk/credential-provider-process@npm:3.972.21" @@ -737,6 +854,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-process@npm:^3.972.70": + version: 3.972.70 + resolution: "@aws-sdk/credential-provider-process@npm:3.972.70" + dependencies: + "@aws-sdk/core": "npm:^3.977.9" + "@aws-sdk/types": "npm:^3.974.5" + "@smithy/core": "npm:^3.33.3" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/87ea5f575f611461a538312b6c502baf86ce33eef6974bb2f9e85bd32147cb8f6b518e80f5a5d8ef371fceb03d517f4d3b6c07ddeb7f15d5a2f56c6dcbf8e8ba + languageName: node + linkType: hard + "@aws-sdk/credential-provider-sso@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-sso@npm:3.972.23" @@ -753,6 +883,21 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-sso@npm:^3.973.14": + version: 3.973.14 + resolution: "@aws-sdk/credential-provider-sso@npm:3.973.14" + dependencies: + "@aws-sdk/core": "npm:^3.977.9" + "@aws-sdk/nested-clients": "npm:^3.997.44" + "@aws-sdk/token-providers": "npm:3.1116.0" + "@aws-sdk/types": "npm:^3.974.5" + "@smithy/core": "npm:^3.33.3" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/12132e57c07277b4c115c835bb7a14b2a62e4f30a69998eca501a171d46be469439b3e7b33c970165362bd261b8689389dd06ba0a959657d97330f3e5172ec52 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-web-identity@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-web-identity@npm:3.972.23" @@ -768,6 +913,42 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-web-identity@npm:^3.972.76": + version: 3.972.76 + resolution: "@aws-sdk/credential-provider-web-identity@npm:3.972.76" + dependencies: + "@aws-sdk/core": "npm:^3.977.9" + "@aws-sdk/nested-clients": "npm:^3.997.44" + "@aws-sdk/types": "npm:^3.974.5" + "@smithy/core": "npm:^3.33.3" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/5efcf6a4b10e49b5b3f9bf76b638ff0b0ce84d26126197f0d28cecbd01458f5d84a074a4cdc8817a7310a940a36a8ad164c1531eb031713bd1485d52bef8c877 + languageName: node + linkType: hard + +"@aws-sdk/dynamodb-codec@npm:^3.973.44": + version: 3.973.44 + resolution: "@aws-sdk/dynamodb-codec@npm:3.973.44" + dependencies: + "@aws-sdk/core": "npm:^3.977.9" + "@smithy/core": "npm:^3.33.3" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/6c7d9c68ac87bf8806b978b3200cfaa81f373ccba466ed43306416ff8a9d794214b5a7c03695ebff83ea818de99d90f9812e41fb0742c5bac0b3fdd308833299 + languageName: node + linkType: hard + +"@aws-sdk/endpoint-cache@npm:^3.972.11": + version: 3.972.11 + resolution: "@aws-sdk/endpoint-cache@npm:3.972.11" + dependencies: + mnemonist: "npm:0.38.3" + tslib: "npm:^2.6.2" + checksum: 10c0/8f3c039fb2dd8e434cd2992bce2a840e6f6470be5ae6d71c762f6cb842149fb60d5566ca470319d5abcf90b115d2c1a0114ed3328afbfc0abed4cbcd52ad03a1 + languageName: node + linkType: hard + "@aws-sdk/lib-storage@npm:^3.1009.0": version: 3.1014.0 resolution: "@aws-sdk/lib-storage@npm:3.1014.0" @@ -800,6 +981,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/middleware-endpoint-discovery@npm:^3.972.30": + version: 3.972.30 + resolution: "@aws-sdk/middleware-endpoint-discovery@npm:3.972.30" + dependencies: + "@aws-sdk/endpoint-cache": "npm:^3.972.11" + "@aws-sdk/types": "npm:^3.974.5" + "@smithy/core": "npm:^3.33.3" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/5655eccf7293f481f45a57359a7d7737ff7345cae3602ae5cb669653f7bc1503f61ce837f11d8bfe399726ae189503cfdc2c21d198816f5acc2d238b7b3476e2 + languageName: node + linkType: hard + "@aws-sdk/middleware-expect-continue@npm:^3.972.8": version: 3.972.8 resolution: "@aws-sdk/middleware-expect-continue@npm:3.972.8" @@ -1006,6 +1200,22 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/nested-clients@npm:^3.997.44": + version: 3.997.44 + resolution: "@aws-sdk/nested-clients@npm:3.997.44" + dependencies: + "@aws-sdk/core": "npm:^3.977.9" + "@aws-sdk/signature-v4-multi-region": "npm:^3.996.46" + "@aws-sdk/types": "npm:^3.974.5" + "@smithy/core": "npm:^3.33.3" + "@smithy/fetch-http-handler": "npm:^5.7.2" + "@smithy/node-http-handler": "npm:^4.11.3" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/947c2049a69a02399f600bd448f34bcaaf3a97b5bbb9b43e9ce932e84d4e9c3131687d330b9c48391da4365582890dd72bf4b8dfafe0693c041e48aa0f5c972d + languageName: node + linkType: hard + "@aws-sdk/region-config-resolver@npm:^3.972.9": version: 3.972.9 resolution: "@aws-sdk/region-config-resolver@npm:3.972.9" @@ -1033,6 +1243,18 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/signature-v4-multi-region@npm:^3.996.46": + version: 3.996.46 + resolution: "@aws-sdk/signature-v4-multi-region@npm:3.996.46" + dependencies: + "@aws-sdk/types": "npm:^3.974.5" + "@smithy/signature-v4": "npm:^5.6.12" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/069dfb7a95663cad2e0aec1d87df8a800abad33cb49dfbe9412dad2d63ab6350328c6165166b12d50e609d8dbe5a5536aa6b1101be4d91584fbca76b2fea4d00 + languageName: node + linkType: hard + "@aws-sdk/token-providers@npm:3.1014.0": version: 3.1014.0 resolution: "@aws-sdk/token-providers@npm:3.1014.0" @@ -1048,6 +1270,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/token-providers@npm:3.1116.0": + version: 3.1116.0 + resolution: "@aws-sdk/token-providers@npm:3.1116.0" + dependencies: + "@aws-sdk/core": "npm:^3.977.9" + "@aws-sdk/nested-clients": "npm:^3.997.44" + "@aws-sdk/types": "npm:^3.974.5" + "@smithy/core": "npm:^3.33.3" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/e720401f6b6d5682984cf1927d3e4c86663b750086bac3b58827a3b9b8585c8b06fb93af82a03a40d5a8ca48c89c0f47c1ac48ea81822c52ddecc3e36ef7cf17 + languageName: node + linkType: hard + "@aws-sdk/types@npm:^3.222.0, @aws-sdk/types@npm:^3.4.1, @aws-sdk/types@npm:^3.973.6": version: 3.973.6 resolution: "@aws-sdk/types@npm:3.973.6" @@ -1058,6 +1294,16 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/types@npm:^3.974.5": + version: 3.974.5 + resolution: "@aws-sdk/types@npm:3.974.5" + dependencies: + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/803aaaa1c0675dcb564803993f3c47d96302fad461af8af80e75afc40c72228ebf669593c18e44ca09fc5acf0d1bd25966261de07844d8f11ad82aa2650252d0 + languageName: node + linkType: hard + "@aws-sdk/util-arn-parser@npm:^3.972.3": version: 3.972.3 resolution: "@aws-sdk/util-arn-parser@npm:3.972.3" @@ -1143,6 +1389,16 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/xml-builder@npm:^3.972.40": + version: 3.972.40 + resolution: "@aws-sdk/xml-builder@npm:3.972.40" + dependencies: + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/5b06fa0466b5ddb0e33138dc16f6a30e11e52fc5ea71f3ed72af7b24621d29c9e8103df3eed9c4f14cef50f4d345dd9e7021242a8c155a67869ee4ce79982bb4 + languageName: node + linkType: hard + "@aws/lambda-invoke-store@npm:0.2.3, @aws/lambda-invoke-store@npm:^0.2.2": version: 0.2.3 resolution: "@aws/lambda-invoke-store@npm:0.2.3" @@ -1150,6 +1406,13 @@ __metadata: languageName: node linkType: hard +"@aws/lambda-invoke-store@npm:^0.3.0": + version: 0.3.0 + resolution: "@aws/lambda-invoke-store@npm:0.3.0" + checksum: 10c0/b4a2e6b3b5397bc606053e64270d26dc5c886336f88a98cad587b1592eec17058f8fb172f1827a9f0e591f3595cf8f01575c8c9b36cde38c06456f8a65204046 + languageName: node + linkType: hard + "@babel/code-frame@npm:^7.0.0, @babel/code-frame@npm:^7.12.13, @babel/code-frame@npm:^7.28.6, @babel/code-frame@npm:^7.29.0": version: 7.29.0 resolution: "@babel/code-frame@npm:7.29.0" @@ -4570,6 +4833,16 @@ __metadata: languageName: node linkType: hard +"@smithy/core@npm:^3.33.2, @smithy/core@npm:^3.33.3": + version: 3.33.3 + resolution: "@smithy/core@npm:3.33.3" + dependencies: + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/57c5c6c1834d84eddfff905932350973afa20e1006024b82d6599af4c8d1e75b243e06b93c998acc95946399f1342ddc298c6941e0030384e6541ceec4007376 + languageName: node + linkType: hard + "@smithy/credential-provider-imds@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/credential-provider-imds@npm:4.2.12" @@ -4583,6 +4856,17 @@ __metadata: languageName: node linkType: hard +"@smithy/credential-provider-imds@npm:^4.4.16": + version: 4.5.2 + resolution: "@smithy/credential-provider-imds@npm:4.5.2" + dependencies: + "@smithy/core": "npm:^3.33.2" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/d5481a7797a1f485849d92f6c188cfb9411736ae2469e27394b863e107dd1024266baa8e3a62cfc3f75b3abfd84b3c389955a0e96b64c4ad90462c70bbe66ab0 + languageName: node + linkType: hard + "@smithy/eventstream-codec@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/eventstream-codec@npm:4.2.12" @@ -4651,6 +4935,17 @@ __metadata: languageName: node linkType: hard +"@smithy/fetch-http-handler@npm:^5.7.2": + version: 5.7.2 + resolution: "@smithy/fetch-http-handler@npm:5.7.2" + dependencies: + "@smithy/core": "npm:^3.33.2" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/9f4541cb743a5207f33d6abd135642a5310cc24f0ed5d46836bf3692901110216f64e09df977488a652762332156fe914b934c780f6a95ef8b1c7d3235c99c7f + languageName: node + linkType: hard + "@smithy/hash-blob-browser@npm:^4.2.13": version: 4.2.13 resolution: "@smithy/hash-blob-browser@npm:4.2.13" @@ -4803,6 +5098,17 @@ __metadata: languageName: node linkType: hard +"@smithy/node-http-handler@npm:^4.11.3": + version: 4.11.3 + resolution: "@smithy/node-http-handler@npm:4.11.3" + dependencies: + "@smithy/core": "npm:^3.33.3" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/b1ec5956281d7ab5a5d1d4ca890b5d9f84f6540bb80c07e5b0eb31426bad27ff03ae56851e95c3fc8efb52ac9d432df86f9735e1be01db7ecdd1559a85683589 + languageName: node + linkType: hard + "@smithy/node-http-handler@npm:^4.5.0": version: 4.5.0 resolution: "@smithy/node-http-handler@npm:4.5.0" @@ -4901,6 +5207,17 @@ __metadata: languageName: node linkType: hard +"@smithy/signature-v4@npm:^5.6.12": + version: 5.7.3 + resolution: "@smithy/signature-v4@npm:5.7.3" + dependencies: + "@smithy/core": "npm:^3.33.3" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/6976142320c7c112ee817f5329d0adc45c1ec101a095a70e4afcb7e3fa9f18cd4f544dcbb481e1932fb877197ec9e1037054bacda422cccfd8de7eed55905319 + languageName: node + linkType: hard + "@smithy/smithy-client@npm:^4.12.7": version: 4.12.7 resolution: "@smithy/smithy-client@npm:4.12.7" @@ -4934,6 +5251,15 @@ __metadata: languageName: node linkType: hard +"@smithy/types@npm:^4.17.2": + version: 4.17.2 + resolution: "@smithy/types@npm:4.17.2" + dependencies: + tslib: "npm:^2.6.2" + checksum: 10c0/7a11f38e6dacdf2247c17bcd50a68ce6200a09dc6a7ecd8d563de94855a25c1da45c4d98f586e3c6fe94727ddbbde6d88ff212ac8eabf7d49779902d1a99a715 + languageName: node + linkType: hard + "@smithy/url-parser@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/url-parser@npm:4.2.12" @@ -9069,6 +9395,15 @@ __metadata: languageName: node linkType: hard +"mnemonist@npm:0.38.3": + version: 0.38.3 + resolution: "mnemonist@npm:0.38.3" + dependencies: + obliterator: "npm:^1.6.1" + checksum: 10c0/064aa1ee1a89fce2754423b3617c598fd65bc34311eb3c01dc063976f6b819b073bd23532415cf8c92240157b4c8fbb7ec5d79d717f2bd4fcd95d8131cb23acb + languageName: node + linkType: hard + "moment-timezone@npm:^0.6.0": version: 0.6.0 resolution: "moment-timezone@npm:0.6.0" @@ -9471,6 +9806,13 @@ __metadata: languageName: node linkType: hard +"obliterator@npm:^1.6.1": + version: 1.6.1 + resolution: "obliterator@npm:1.6.1" + checksum: 10c0/5fad57319aae0ef6e34efa640541d41c2dd9790a7ab808f17dcb66c83a81333963fc2dfcfa6e1b62158e5cef6291cdcf15c503ad6c3de54b2227dd4c3d7e1b55 + languageName: node + linkType: hard + "obug@npm:^2.1.1": version: 2.1.1 resolution: "obug@npm:2.1.1"