From 19321918adbc92117bd5332b031714dd80e41a3b Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Wed, 19 Aug 2026 11:02:53 +0200 Subject: [PATCH 1/4] refactor(storage): extract webhook matcher config --- lambdas/functions/webhook/package.json | 1 + .../webhook/src/ConfigLoader.test.ts | 163 +++++------------- lambdas/functions/webhook/src/ConfigLoader.ts | 42 +---- lambdas/functions/webhook/src/lambda.test.ts | 15 +- lambdas/functions/webhook/src/modules.d.ts | 1 - .../webhook/src/runners/dispatch.test.ts | 26 ++- .../webhook/src/webhook/index.test.ts | 22 ++- .../aws/ssm/environment.d.ts | 1 + .../ssm/runner-matcher-config-store.test.ts | 103 +++++++++++ .../aws/ssm/runner-matcher-config-store.ts | 69 ++++++++ lambdas/libs/storage-providers/core/index.ts | 4 + lambdas/libs/storage-providers/index.ts | 2 + .../runner-matcher-config.test.ts | 83 +++++++++ .../runner-matcher-config.ts | 23 +++ .../libs/storage-providers/vitest.config.ts | 1 + lambdas/yarn.lock | 1 + 16 files changed, 369 insertions(+), 188 deletions(-) create mode 100644 lambdas/libs/storage-providers/aws/ssm/runner-matcher-config-store.test.ts create mode 100644 lambdas/libs/storage-providers/aws/ssm/runner-matcher-config-store.ts create mode 100644 lambdas/libs/storage-providers/runner-matcher-config.test.ts create mode 100644 lambdas/libs/storage-providers/runner-matcher-config.ts diff --git a/lambdas/functions/webhook/package.json b/lambdas/functions/webhook/package.json index 34f4ef3de9..83d9810d1f 100644 --- a/lambdas/functions/webhook/package.json +++ b/lambdas/functions/webhook/package.json @@ -31,6 +31,7 @@ "@aws-github-runner/aws-powertools-util": "*", "@aws-github-runner/aws-ssm-util": "*", "@aws-github-runner/compute-providers": "*", + "@aws-github-runner/storage-providers": "*", "@aws-sdk/client-sqs": "^3.1009.0", "@middy/core": "^6.4.5", "@octokit/rest": "22.0.1", diff --git a/lambdas/functions/webhook/src/ConfigLoader.test.ts b/lambdas/functions/webhook/src/ConfigLoader.test.ts index 9f4e5e5864..7c7aa1dcf7 100644 --- a/lambdas/functions/webhook/src/ConfigLoader.test.ts +++ b/lambdas/functions/webhook/src/ConfigLoader.test.ts @@ -1,4 +1,5 @@ -import { getParameter, getParameters } from '@aws-github-runner/aws-ssm-util'; +import { getParameter } from '@aws-github-runner/aws-ssm-util'; +import { getRunnerMatcherConfigStore, type RunnerMatcherConfigStore } from '@aws-github-runner/storage-providers'; import { ConfigWebhook, ConfigWebhookEventBridge, ConfigDispatcher } from './ConfigLoader'; import { logger } from '@aws-github-runner/aws-powertools-util'; @@ -6,6 +7,11 @@ import { RunnerMatcherConfig } from './sqs'; import { describe, it, expect, beforeEach, vi } from 'vitest'; vi.mock('@aws-github-runner/aws-ssm-util'); +vi.mock('@aws-github-runner/storage-providers'); + +const runnerMatcherConfigStore = { + get: vi.fn(), +} satisfies RunnerMatcherConfigStore; describe('ConfigLoader Tests', () => { beforeEach(() => { @@ -14,6 +20,7 @@ describe('ConfigLoader Tests', () => { ConfigWebhookEventBridge.reset(); ConfigDispatcher.reset(); logger.setLogLevel('DEBUG'); + vi.mocked(getRunnerMatcherConfigStore).mockReturnValue(runnerMatcherConfigStore); // clear process.env for (const key of Object.keys(process.env)) { @@ -24,7 +31,6 @@ describe('ConfigLoader Tests', () => { describe('Check base object', () => { function setupConfiguration(): void { process.env.EVENT_BUS_NAME = 'event-bus'; - process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config'; process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; const matcherConfig = [ { @@ -36,15 +42,8 @@ describe('ConfigLoader Tests', () => { }, }, ]; - vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { - if (paramPath === '/path/to/matcher/config') { - return JSON.stringify(matcherConfig); - } - if (paramPath === '/path/to/webhook/secret') { - return 'secret'; - } - return ''; - }); + runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(matcherConfig)); + vi.mocked(getParameter).mockResolvedValue('secret'); } it('should return the same instance of ConfigWebhook (singleton)', async () => { @@ -53,7 +52,8 @@ describe('ConfigLoader Tests', () => { const config2 = await ConfigWebhook.load(); expect(config1).toBe(config2); - expect(getParameter).toHaveBeenCalledTimes(2); + expect(getParameter).toHaveBeenCalledOnce(); + expect(runnerMatcherConfigStore.get).toHaveBeenCalledOnce(); }); it('should return the same instance of ConfigWebhookEventBridge (singleton)', async () => { @@ -63,6 +63,7 @@ describe('ConfigLoader Tests', () => { expect(config1).toBe(config2); expect(getParameter).toHaveBeenCalledTimes(1); + expect(runnerMatcherConfigStore.get).not.toHaveBeenCalled(); }); it('should return the same instance of ConfigDispatcher (singleton)', async () => { @@ -71,7 +72,8 @@ describe('ConfigLoader Tests', () => { const config2 = await ConfigDispatcher.load(); expect(config1).toBe(config2); - expect(getParameter).toHaveBeenCalledTimes(1); + expect(getParameter).not.toHaveBeenCalled(); + expect(runnerMatcherConfigStore.get).toHaveBeenCalledOnce(); }); it('should filter secrets from being logged', async () => { @@ -95,7 +97,6 @@ describe('ConfigLoader Tests', () => { it('should load config successfully', async () => { process.env.REPOSITORY_ALLOW_LIST = '["repo1", "repo2"]'; process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; - process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config'; const matcherConfig = [ { id: '1', @@ -106,15 +107,8 @@ describe('ConfigLoader Tests', () => { }, }, ]; - vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { - if (paramPath === '/path/to/matcher/config') { - return JSON.stringify(matcherConfig); - } - if (paramPath === '/path/to/webhook/secret') { - return 'secret'; - } - return ''; - }); + runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(matcherConfig)); + vi.mocked(getParameter).mockResolvedValue('secret'); const config: ConfigWebhook = await ConfigWebhook.load(); @@ -124,7 +118,6 @@ describe('ConfigLoader Tests', () => { }); it('should load config successfully', async () => { - process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config'; process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; const matcherConfig = [ { @@ -136,15 +129,8 @@ describe('ConfigLoader Tests', () => { }, }, ]; - vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { - if (paramPath === '/path/to/matcher/config') { - return JSON.stringify(matcherConfig); - } - if (paramPath === '/path/to/webhook/secret') { - return 'secret'; - } - return ''; - }); + runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(matcherConfig)); + vi.mocked(getParameter).mockResolvedValue('secret'); const config: ConfigWebhook = await ConfigWebhook.load(); @@ -155,46 +141,27 @@ describe('ConfigLoader Tests', () => { }); it('should throw error if config loading fails', async () => { - process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config'; - - vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { - if (paramPath === '/path/to/matcher/config') { - throw new Error('Failed to load matcher config'); - } - return ''; - }); + runnerMatcherConfigStore.get.mockRejectedValue( + new Error( + 'Failed to load parameter for matcherConfig from path /path/to/matcher/config: Failed to load matcher config', + ), + ); + vi.mocked(getParameter).mockResolvedValue(''); await expect(ConfigWebhook.load()).rejects.toThrow( 'Failed to load config: Failed to load parameter for matcherConfig from path /path/to/matcher/config: Failed to load matcher config', ); }); - it('should load config successfully from multiple paths', async () => { - process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config-1:/path/to/matcher/config-2'; + it('should load combined matcher config returned by the store', async () => { process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; - const partialMatcher1 = - '[{"id":"1","arn":"arn:aws:sqs:queue1","matcherConfig":{"labelMatchers":[["a"]],"exactMatch":true}}'; - const partialMatcher2 = - ',{"id":"2","arn":"arn:aws:sqs:queue2","matcherConfig":{"labelMatchers":[["b"]],"exactMatch":true}}]'; - const combinedMatcherConfig = [ { id: '1', arn: 'arn:aws:sqs:queue1', matcherConfig: { labelMatchers: [['a']], exactMatch: true } }, { id: '2', arn: 'arn:aws:sqs:queue2', matcherConfig: { labelMatchers: [['b']], exactMatch: true } }, ]; - - // Mock getParameters for batch fetching multiple paths - vi.mocked(getParameters).mockResolvedValue( - new Map([ - ['/path/to/matcher/config-1', partialMatcher1], - ['/path/to/matcher/config-2', partialMatcher2], - ]), - ); - - vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { - if (paramPath === '/path/to/webhook/secret') return 'secret'; - return ''; - }); + runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(combinedMatcherConfig)); + vi.mocked(getParameter).mockResolvedValue('secret'); const config: ConfigWebhook = await ConfigWebhook.load(); @@ -202,27 +169,14 @@ describe('ConfigLoader Tests', () => { expect(config.webhookSecret).toBe('secret'); }); - it('should throw error if config loading fails from multiple paths', async () => { - process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config-1:/path/to/matcher/config-2'; + it('should propagate an error from the matcher config store', async () => { process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; - - const partialMatcher1 = - '[{"id":"1","arn":"arn:aws:sqs:queue1","matcherConfig":{"labelMatchers":[["a"]],"exactMatch":true}}'; - const partialMatcher2 = - ',{"id":"2","arn":"arn:aws:sqs:queue2","matcherConfig":{"labelMatchers":[["b"]],"exactMatch":true}}'; - - // Mock getParameters for batch fetching - returns incomplete JSON that will fail to parse - vi.mocked(getParameters).mockResolvedValue( - new Map([ - ['/path/to/matcher/config-1', partialMatcher1], - ['/path/to/matcher/config-2', partialMatcher2], - ]), + runnerMatcherConfigStore.get.mockRejectedValue( + new Error( + "Failed to load/parse combined matcher config: Expected ',' or ']' after array element in JSON at position 196", + ), ); - - vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { - if (paramPath === '/path/to/webhook/secret') return 'secret'; - return ''; - }); + vi.mocked(getParameter).mockResolvedValue('secret'); await expect(ConfigWebhook.load()).rejects.toThrow( "Failed to load config: Failed to load/parse combined matcher config: Expected ',' or ']' after array element in JSON at position 196", @@ -248,6 +202,7 @@ describe('ConfigLoader Tests', () => { expect(config.allowedEvents).toEqual(['push', 'pull_request']); expect(config.eventBusName).toBe('event-bus'); expect(config.webhookSecret).toBe('secret'); + expect(runnerMatcherConfigStore.get).not.toHaveBeenCalled(); }); it('should throw error if config loading fails', async () => { @@ -264,7 +219,6 @@ describe('ConfigLoader Tests', () => { describe('ConfigDispatcher', () => { it('should load config successfully', async () => { process.env.REPOSITORY_ALLOW_LIST = '["repo1", "repo2"]'; - process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config'; const matcherConfig: RunnerMatcherConfig[] = [ { @@ -276,12 +230,7 @@ describe('ConfigLoader Tests', () => { }, }, ]; - vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { - if (paramPath === '/path/to/matcher/config') { - return JSON.stringify(matcherConfig); - } - return ''; - }); + runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(matcherConfig)); const config: ConfigDispatcher = await ConfigDispatcher.load(); @@ -289,27 +238,14 @@ describe('ConfigLoader Tests', () => { expect(config.matcherConfig).toEqual(matcherConfig); }); - it('should load config successfully from multiple paths with repo allow list', async () => { + it('should load combined matcher config returned by the store with repo allow list', async () => { process.env.REPOSITORY_ALLOW_LIST = '["repo1", "repo2"]'; - process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config-1:/path/to/matcher/config-2'; - - const partial1 = - '[{"id":"1","arn":"arn:aws:sqs:queue1","matcherConfig":{"labelMatchers":[["x"]],"exactMatch":true}}'; - const partial2 = - ',{"id":"2","arn":"arn:aws:sqs:queue2","matcherConfig":{"labelMatchers":[["y"]],"exactMatch":true}}]'; const combined: RunnerMatcherConfig[] = [ { id: '1', arn: 'arn:aws:sqs:queue1', matcherConfig: { labelMatchers: [['x']], exactMatch: true } }, { id: '2', arn: 'arn:aws:sqs:queue2', matcherConfig: { labelMatchers: [['y']], exactMatch: true } }, ]; - - // Mock getParameters for batch fetching multiple paths - vi.mocked(getParameters).mockResolvedValue( - new Map([ - ['/path/to/matcher/config-1', partial1], - ['/path/to/matcher/config-2', partial2], - ]), - ); + runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(combined)); const config: ConfigDispatcher = await ConfigDispatcher.load(); @@ -318,18 +254,15 @@ describe('ConfigLoader Tests', () => { }); it('should throw error if config loading fails', async () => { - vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { - throw new Error(`Parameter ${paramPath} not found`); - }); + runnerMatcherConfigStore.get.mockRejectedValue(new Error('Matcher config store is unavailable')); await expect(ConfigDispatcher.load()).rejects.toThrow( - 'Failed to load config: Failed to load parameter for matcherConfig from path undefined: Parameter undefined not found', + 'Failed to load config: Matcher config store is unavailable', ); }); it('should rely on default when optionals are not set.', async () => { process.env.ACCEPT_EVENTS = 'null'; - process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config'; const matcherConfig: RunnerMatcherConfig[] = [ { arn: 'arn:aws:sqs:eu-central-1:123456:npalm-default-queued-builds', @@ -340,12 +273,7 @@ describe('ConfigLoader Tests', () => { }, }, ]; - vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { - if (paramPath === '/path/to/matcher/config') { - return JSON.stringify(matcherConfig); - } - return ''; - }); + runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(matcherConfig)); const config: ConfigDispatcher = await ConfigDispatcher.load(); @@ -355,14 +283,7 @@ describe('ConfigLoader Tests', () => { it('should throw an error if runner matcher config is empty.', async () => { process.env.REPOSITORY_ALLOW_LIST = '["repo1", "repo2"]'; - process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config'; - - vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { - if (paramPath === '/path/to/matcher/config') { - return JSON.stringify(''); - } - return ''; - }); + runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify('')); await expect(ConfigDispatcher.load()).rejects.toThrow('Failed to load config: Matcher config is empty'); }); diff --git a/lambdas/functions/webhook/src/ConfigLoader.ts b/lambdas/functions/webhook/src/ConfigLoader.ts index d9d9da2590..2cf261b849 100644 --- a/lambdas/functions/webhook/src/ConfigLoader.ts +++ b/lambdas/functions/webhook/src/ConfigLoader.ts @@ -1,4 +1,5 @@ -import { getParameter, getParameters } from '@aws-github-runner/aws-ssm-util'; +import { getParameter } from '@aws-github-runner/aws-ssm-util'; +import { getRunnerMatcherConfigStore } from '@aws-github-runner/storage-providers'; import { RunnerMatcherConfig } from './sqs'; import { logger } from '@aws-github-runner/aws-powertools-util'; @@ -66,7 +67,7 @@ abstract class BaseConfig { }); } - private loadProperty(propertyName: keyof this, value: string) { + protected loadProperty(propertyName: keyof this, value: string) { try { this[propertyName] = JSON.parse(value) as unknown as this[keyof this]; } catch { @@ -96,38 +97,11 @@ abstract class MatcherAwareConfig extends BaseConfig { // across the matching queues to avoid concentrating load on a single one. queueSelectionStrategy: QueueSelectionStrategy = 'first'; - protected async loadMatcherConfig(paramPathsEnv: string) { - if (!paramPathsEnv || paramPathsEnv === 'undefined' || paramPathsEnv === 'null' || !paramPathsEnv.includes(':')) { - // Single path or invalid string → load directly - await this.loadParameter(paramPathsEnv, 'matcherConfig'); - return; - } - - const paths = paramPathsEnv - .split(':') - .map((p) => p.trim()) - .filter(Boolean); - - // Batch fetch all matcher config paths in a single SSM API call + protected async loadMatcherConfig() { try { - const params = await getParameters(paths); - let combinedString = ''; - for (const path of paths) { - const value = params.get(path); - if (value) { - combinedString += value; - } else { - this.configLoadingErrors.push( - `Failed to load parameter for matcherConfig from path ${path}: Parameter not found`, - ); - } - } - - if (combinedString) { - this.matcherConfig = JSON.parse(combinedString); - } + this.loadProperty('matcherConfig', await getRunnerMatcherConfigStore().get()); } catch (error) { - this.configLoadingErrors.push(`Failed to load/parse combined matcher config: ${(error as Error).message}`); + this.configLoadingErrors.push((error as Error).message); } } } @@ -142,7 +116,7 @@ export class ConfigWebhook extends MatcherAwareConfig { this.loadEnvVar(process.env.QUEUE_SELECTION_STRATEGY, 'queueSelectionStrategy', 'first'); await Promise.all([ - this.loadMatcherConfig(process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH), + this.loadMatcherConfig(), this.loadParameter(process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET, 'webhookSecret'), ]); @@ -174,7 +148,7 @@ export class ConfigDispatcher extends MatcherAwareConfig { async loadConfig(): Promise { this.loadEnvVar(process.env.REPOSITORY_ALLOW_LIST, 'repositoryAllowList', []); this.loadEnvVar(process.env.QUEUE_SELECTION_STRATEGY, 'queueSelectionStrategy', 'first'); - await this.loadMatcherConfig(process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH); + await this.loadMatcherConfig(); validateRunnerMatcherConfig(this); validateQueueSelectionStrategy(this); diff --git a/lambdas/functions/webhook/src/lambda.test.ts b/lambdas/functions/webhook/src/lambda.test.ts index d65b8371c4..3bc67e42dd 100644 --- a/lambdas/functions/webhook/src/lambda.test.ts +++ b/lambdas/functions/webhook/src/lambda.test.ts @@ -7,6 +7,7 @@ import { dispatchToRunners, eventBridgeWebhook, directWebhook } from './lambda'; import { publishForRunners, publishOnEventBridge } from './webhook'; import ValidationError from './ValidationError'; import { getParameter } from '@aws-github-runner/aws-ssm-util'; +import { getRunnerMatcherConfigStore, type RunnerMatcherConfigStore } from '@aws-github-runner/storage-providers'; import { dispatch } from './runners/dispatch'; import { EventWrapper } from './types'; import { describe, it, expect, beforeEach, vi } from 'vitest'; @@ -80,14 +81,20 @@ const context: Context = { vi.mock('./runners/dispatch'); vi.mock('./webhook'); vi.mock('@aws-github-runner/aws-ssm-util'); +vi.mock('@aws-github-runner/storage-providers'); + +const runnerMatcherConfigStore = { + get: vi.fn(), +} satisfies RunnerMatcherConfigStore; describe('Test webhook lambda wrapper.', () => { beforeEach(() => { - // We mock all SSM request to resolve to a non empty array. Since we mock all implemeantions - // relying on the config object that is enough to test the handlers. - const mockedGet = vi.mocked(getParameter); - mockedGet.mockResolvedValue('["abc"]'); vi.clearAllMocks(); + // The handlers only need non-empty config values because their downstream + // implementations are mocked in this wrapper test. + vi.mocked(getParameter).mockResolvedValue('["abc"]'); + vi.mocked(getRunnerMatcherConfigStore).mockReturnValue(runnerMatcherConfigStore); + runnerMatcherConfigStore.get.mockResolvedValue('["abc"]'); }); describe('Test webhook lambda wrapper.', () => { diff --git a/lambdas/functions/webhook/src/modules.d.ts b/lambdas/functions/webhook/src/modules.d.ts index 9110746709..3b04a2a5be 100644 --- a/lambdas/functions/webhook/src/modules.d.ts +++ b/lambdas/functions/webhook/src/modules.d.ts @@ -3,7 +3,6 @@ declare namespace NodeJS { ENVIRONMENT: string; EVENT_BUS_NAME: string; PARAMETER_GITHUB_APP_WEBHOOK_SECRET: string; - PARAMETER_RUNNER_MATCHER_CONFIG_PATH: string; QUEUE_SELECTION_STRATEGY: string; REPOSITORY_ALLOW_LIST: string; RUNNER_LABELS: string; diff --git a/lambdas/functions/webhook/src/runners/dispatch.test.ts b/lambdas/functions/webhook/src/runners/dispatch.test.ts index bb2cdc7cce..b3140d6e96 100644 --- a/lambdas/functions/webhook/src/runners/dispatch.test.ts +++ b/lambdas/functions/webhook/src/runners/dispatch.test.ts @@ -1,5 +1,5 @@ -import { getParameter } from '@aws-github-runner/aws-ssm-util'; import { selectDynamicLabelQueue } from '@aws-github-runner/compute-providers/webhook'; +import { getRunnerMatcherConfigStore, type RunnerMatcherConfigStore } from '@aws-github-runner/storage-providers'; import nock from 'nock'; import { WorkflowJobEvent } from '@octokit/webhooks-types'; @@ -14,12 +14,14 @@ import { logger } from '@aws-github-runner/aws-powertools-util'; import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; vi.mock('../sqs'); -vi.mock('@aws-github-runner/aws-ssm-util'); +vi.mock('@aws-github-runner/storage-providers'); vi.mock('@aws-github-runner/compute-providers/webhook', () => ({ selectDynamicLabelQueue: vi.fn(), })); -const GITHUB_APP_WEBHOOK_SECRET = 'TEST_SECRET'; +const runnerMatcherConfigStore = { + get: vi.fn(), +} satisfies RunnerMatcherConfigStore; const cleanEnv = process.env; @@ -37,7 +39,6 @@ describe('Dispatcher', () => { vi.clearAllMocks(); vi.resetAllMocks(); - mockSSMResponse(); config = await createConfig(undefined, runnerConfig); }); @@ -242,7 +243,7 @@ describe('Dispatcher', () => { it('rejects an invalid strategy at config load', async () => { process.env.QUEUE_SELECTION_STRATEGY = 'bogus'; ConfigDispatcher.reset(); - mockSSMResponse(twoExactMatches); + mockMatcherConfigResponse(twoExactMatches); await expect(ConfigDispatcher.load()).rejects.toThrow(/queue selection strategy/i); }); }); @@ -394,16 +395,9 @@ describe('Dispatcher', () => { }); }); -function mockSSMResponse(runnerConfigInput?: RunnerConfig) { - process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/github-runner/runner-matcher-config'; - const mockedGet = vi.mocked(getParameter); - mockedGet.mockImplementation((parameter_name) => { - const value = - parameter_name == '/github-runner/runner-matcher-config' - ? JSON.stringify(runnerConfigInput ?? runnerConfig) - : GITHUB_APP_WEBHOOK_SECRET; - return Promise.resolve(value); - }); +function mockMatcherConfigResponse(runnerConfigInput?: RunnerConfig) { + vi.mocked(getRunnerMatcherConfigStore).mockReturnValue(runnerMatcherConfigStore); + runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(runnerConfigInput ?? runnerConfig)); } async function createConfig(repositoryAllowList?: string[], runnerConfig?: RunnerConfig): Promise { @@ -411,6 +405,6 @@ async function createConfig(repositoryAllowList?: string[], runnerConfig?: Runne process.env.REPOSITORY_ALLOW_LIST = JSON.stringify(repositoryAllowList); } ConfigDispatcher.reset(); - mockSSMResponse(runnerConfig); + mockMatcherConfigResponse(runnerConfig); return await ConfigDispatcher.load(); } diff --git a/lambdas/functions/webhook/src/webhook/index.test.ts b/lambdas/functions/webhook/src/webhook/index.test.ts index aa4fbbc506..43345388f7 100644 --- a/lambdas/functions/webhook/src/webhook/index.test.ts +++ b/lambdas/functions/webhook/src/webhook/index.test.ts @@ -1,5 +1,6 @@ import { Webhooks } from '@octokit/webhooks'; import { getParameter } from '@aws-github-runner/aws-ssm-util'; +import { getRunnerMatcherConfigStore, type RunnerMatcherConfigStore } from '@aws-github-runner/storage-providers'; import nock from 'nock'; @@ -16,8 +17,12 @@ vi.mock('../sqs'); vi.mock('../eventbridge'); vi.mock('../runners/dispatch'); vi.mock('@aws-github-runner/aws-ssm-util'); +vi.mock('@aws-github-runner/storage-providers'); const GITHUB_APP_WEBHOOK_SECRET = 'TEST_SECRET'; +const runnerMatcherConfigStore = { + get: vi.fn(), +} satisfies RunnerMatcherConfigStore; const cleanEnv = process.env; @@ -32,7 +37,7 @@ describe('handle GitHub webhook events', () => { nock.disableNetConnect(); vi.clearAllMocks(); - mockSSMResponse(); + mockConfigResponse(); }); describe('handle and dispatch webhook events to build queues', () => { @@ -284,8 +289,7 @@ describe('Check message size (checkBodySize)', () => { }); }); -function mockSSMResponse() { - process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/path/to/matcher/config'; +function mockConfigResponse() { process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; const matcherConfig = [ { @@ -297,13 +301,7 @@ function mockSSMResponse() { }, }, ]; - vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { - if (paramPath === '/path/to/matcher/config') { - return JSON.stringify(matcherConfig); - } - if (paramPath === '/path/to/webhook/secret') { - return GITHUB_APP_WEBHOOK_SECRET; - } - throw new Error('Parameter not found'); - }); + vi.mocked(getRunnerMatcherConfigStore).mockReturnValue(runnerMatcherConfigStore); + runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(matcherConfig)); + vi.mocked(getParameter).mockResolvedValue(GITHUB_APP_WEBHOOK_SECRET); } diff --git a/lambdas/libs/storage-providers/aws/ssm/environment.d.ts b/lambdas/libs/storage-providers/aws/ssm/environment.d.ts index 2604ce057e..5ffa7b85bb 100644 --- a/lambdas/libs/storage-providers/aws/ssm/environment.d.ts +++ b/lambdas/libs/storage-providers/aws/ssm/environment.d.ts @@ -3,6 +3,7 @@ export {}; declare global { namespace NodeJS { interface ProcessEnv { + PARAMETER_RUNNER_MATCHER_CONFIG_PATH?: string; SSM_PARAMETER_STORE_TAGS?: string; SSM_CONFIG_PATH?: string; SSM_TOKEN_PATH?: string; diff --git a/lambdas/libs/storage-providers/aws/ssm/runner-matcher-config-store.test.ts b/lambdas/libs/storage-providers/aws/ssm/runner-matcher-config-store.test.ts new file mode 100644 index 0000000000..4b9d2f3379 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm/runner-matcher-config-store.test.ts @@ -0,0 +1,103 @@ +import { getParameter, getParameters } from '@aws-github-runner/aws-ssm-util'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAwsSsmRunnerMatcherConfigStore } from './runner-matcher-config-store'; + +vi.mock('@aws-github-runner/aws-ssm-util', () => ({ + getParameter: vi.fn(), + getParameters: vi.fn(), +})); + +const getParameterMock = vi.mocked(getParameter); +const getParametersMock = vi.mocked(getParameters); +const cleanEnv = process.env; + +describe('aws_ssm runner matcher config store', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env = { ...cleanEnv }; + delete process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH; + }); + + it('loads a single matcher config parameter', async () => { + process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/runner/matcher/config'; + getParameterMock.mockResolvedValue('[{"id":"runner"}]'); + + await expect(createAwsSsmRunnerMatcherConfigStore().get()).resolves.toBe('[{"id":"runner"}]'); + + expect(getParameterMock).toHaveBeenCalledWith('/runner/matcher/config'); + expect(getParametersMock).not.toHaveBeenCalled(); + }); + + it('loads and concatenates matcher config chunks in configured order', async () => { + process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = ' /runner/matcher/1 : : /runner/matcher/2 '; + getParametersMock.mockResolvedValue( + new Map([ + ['/runner/matcher/2', ',{"id":"runner-2"}]'], + ['/runner/matcher/1', '[{"id":"runner-1"}'], + ]), + ); + + await expect(createAwsSsmRunnerMatcherConfigStore().get()).resolves.toBe('[{"id":"runner-1"},{"id":"runner-2"}]'); + + expect(getParametersMock).toHaveBeenCalledWith(['/runner/matcher/1', '/runner/matcher/2']); + expect(getParameterMock).not.toHaveBeenCalled(); + }); + + it('rejects a missing matcher config chunk', async () => { + process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/runner/matcher/1:/runner/matcher/2'; + getParametersMock.mockResolvedValue(new Map([['/runner/matcher/1', '[{"id":"runner-1"}']])); + + await expect(createAwsSsmRunnerMatcherConfigStore().get()).rejects.toThrow( + 'Failed to load parameter for matcherConfig from path /runner/matcher/2: Parameter not found', + ); + }); + + it('rejects malformed combined matcher config', async () => { + process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/runner/matcher/1:/runner/matcher/2'; + getParametersMock.mockResolvedValue( + new Map([ + ['/runner/matcher/1', '[{"id":"runner-1"}'], + ['/runner/matcher/2', ',{"id":"runner-2"}'], + ]), + ); + + await expect(createAwsSsmRunnerMatcherConfigStore().get()).rejects.toThrow( + "Failed to load/parse combined matcher config: Expected ',' or ']' after array element", + ); + }); + + it('propagates a single parameter read failure', async () => { + process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/runner/matcher/config'; + const error = new Error('read failed'); + getParameterMock.mockRejectedValue(error); + + await expect(createAwsSsmRunnerMatcherConfigStore().get()).rejects.toThrow( + 'Failed to load parameter for matcherConfig from path /runner/matcher/config: read failed', + ); + }); + + it('propagates a batch parameter read failure', async () => { + process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = '/runner/matcher/1:/runner/matcher/2'; + const error = new Error('read failed'); + getParametersMock.mockRejectedValue(error); + + await expect(createAwsSsmRunnerMatcherConfigStore().get()).rejects.toThrow( + 'Failed to load/parse combined matcher config: read failed', + ); + }); + + it.each([undefined, '', ' '])('requires matcher config parameter paths for input %j', (parameterPaths) => { + if (parameterPaths === undefined) { + delete process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH; + } else { + process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH = parameterPaths; + } + + expect(() => createAwsSsmRunnerMatcherConfigStore()).toThrow( + 'Environment variable PARAMETER_RUNNER_MATCHER_CONFIG_PATH is not set', + ); + expect(getParameterMock).not.toHaveBeenCalled(); + expect(getParametersMock).not.toHaveBeenCalled(); + }); +}); diff --git a/lambdas/libs/storage-providers/aws/ssm/runner-matcher-config-store.ts b/lambdas/libs/storage-providers/aws/ssm/runner-matcher-config-store.ts new file mode 100644 index 0000000000..166151b850 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm/runner-matcher-config-store.ts @@ -0,0 +1,69 @@ +import { getParameter, getParameters } from '@aws-github-runner/aws-ssm-util'; + +import type { RunnerMatcherConfigStore } from '../../core'; +import type {} from './environment'; + +export function createAwsSsmRunnerMatcherConfigStore(): RunnerMatcherConfigStore { + const parameterPaths = process.env.PARAMETER_RUNNER_MATCHER_CONFIG_PATH; + if (!parameterPaths || parameterPaths.trim() === '') { + throw new Error('Environment variable PARAMETER_RUNNER_MATCHER_CONFIG_PATH is not set'); + } + + const paths = parameterPaths + .split(':') + .map((path) => path.trim()) + .filter(Boolean); + + if (paths.length === 0) { + throw new Error('Environment variable PARAMETER_RUNNER_MATCHER_CONFIG_PATH is not set'); + } + + return new AwsSsmRunnerMatcherConfigStore(paths); +} + +class AwsSsmRunnerMatcherConfigStore implements RunnerMatcherConfigStore { + constructor(private readonly parameterPaths: string[]) {} + + async get(): Promise { + if (this.parameterPaths.length === 1) { + const path = this.parameterPaths[0]; + try { + return await getParameter(path); + } catch (error) { + throw new Error(`Failed to load parameter for matcherConfig from path ${path}: ${(error as Error).message}`); + } + } + + let parameters: Map; + try { + parameters = await getParameters(this.parameterPaths); + } catch (error) { + throw new Error(`Failed to load/parse combined matcher config: ${(error as Error).message}`); + } + + let combined = ''; + const errors: string[] = []; + for (const path of this.parameterPaths) { + const value = parameters.get(path); + if (value) { + combined += value; + } else { + errors.push(`Failed to load parameter for matcherConfig from path ${path}: Parameter not found`); + } + } + + if (combined) { + try { + JSON.parse(combined); + } catch (error) { + errors.push(`Failed to load/parse combined matcher config: ${(error as Error).message}`); + } + } + + if (errors.length > 0) { + throw new Error(errors.join(', ')); + } + + return combined; + } +} diff --git a/lambdas/libs/storage-providers/core/index.ts b/lambdas/libs/storage-providers/core/index.ts index 27339a2159..3d9fc43fe8 100644 --- a/lambdas/libs/storage-providers/core/index.ts +++ b/lambdas/libs/storage-providers/core/index.ts @@ -58,3 +58,7 @@ export interface RunnerGroupCacheStore { get(runnerGroupName: string): Promise; create(record: RunnerGroupCacheRecord): Promise; } + +export interface RunnerMatcherConfigStore { + get(): Promise; +} diff --git a/lambdas/libs/storage-providers/index.ts b/lambdas/libs/storage-providers/index.ts index 4b5f14f1c6..026b79d48a 100644 --- a/lambdas/libs/storage-providers/index.ts +++ b/lambdas/libs/storage-providers/index.ts @@ -9,6 +9,7 @@ export type { RunnerConfigStore, RunnerGroupCacheRecord, RunnerGroupCacheStore, + RunnerMatcherConfigStore, } from './core'; export { createRunnerConfigHousekeeper } from './runner-config-housekeeper'; export { createRunnerConfigConsumer, type RunnerConfigConsumerConfig } from './runner-config-consumer'; @@ -20,3 +21,4 @@ export { export type { RunnerConfigStorageProvider } from './provider'; export { createCommonStorage, createStorageProviders } from './storage-providers'; export type { StorageProviders, RunnerConfigStorage, CommonStorage } from './core'; +export { getRunnerMatcherConfigStore, resetRunnerMatcherConfigStore } from './runner-matcher-config'; diff --git a/lambdas/libs/storage-providers/runner-matcher-config.test.ts b/lambdas/libs/storage-providers/runner-matcher-config.test.ts new file mode 100644 index 0000000000..0dd7f42ad3 --- /dev/null +++ b/lambdas/libs/storage-providers/runner-matcher-config.test.ts @@ -0,0 +1,83 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAwsSsmRunnerMatcherConfigStore } from './aws/ssm/runner-matcher-config-store'; +import type { RunnerMatcherConfigStore } from './core'; +import { getRunnerMatcherConfigStore, resetRunnerMatcherConfigStore } from './runner-matcher-config'; + +vi.mock('./aws/ssm/runner-matcher-config-store', () => ({ + createAwsSsmRunnerMatcherConfigStore: vi.fn(), +})); + +const createAwsSsmRunnerMatcherConfigStoreMock = vi.mocked(createAwsSsmRunnerMatcherConfigStore); +const cleanEnv = process.env; + +describe('runner matcher config store selection', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env = { ...cleanEnv }; + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + resetRunnerMatcherConfigStore(); + }); + + it.each([undefined, '', ' '])('uses aws_ssm for default selector input %j', (provider) => { + setProvider(provider); + const store = stubStore(); + + expect(getRunnerMatcherConfigStore()).toBe(store); + expect(createAwsSsmRunnerMatcherConfigStoreMock).toHaveBeenCalledOnce(); + }); + + it.each(['aws_ssm', ' AWS_SSM '])('uses aws_ssm for explicit selector input %j', (provider) => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + const store = stubStore(); + + expect(getRunnerMatcherConfigStore()).toBe(store); + expect(createAwsSsmRunnerMatcherConfigStoreMock).toHaveBeenCalledOnce(); + }); + + it('rejects an unsupported provider on first use', () => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = 'not-registered'; + + expect(() => getRunnerMatcherConfigStore()).toThrow("Unsupported runner config storage provider 'not-registered'"); + expect(createAwsSsmRunnerMatcherConfigStoreMock).not.toHaveBeenCalled(); + }); + + it('selects lazily and caches the created store', () => { + const store = stubStore(); + + expect(createAwsSsmRunnerMatcherConfigStoreMock).not.toHaveBeenCalled(); + const first = getRunnerMatcherConfigStore(); + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = 'not-registered'; + const second = getRunnerMatcherConfigStore(); + + expect(first).toBe(store); + expect(second).toBe(store); + expect(createAwsSsmRunnerMatcherConfigStoreMock).toHaveBeenCalledOnce(); + }); + + it('selects again after the test reset', () => { + const firstStore = stubStore(); + expect(getRunnerMatcherConfigStore()).toBe(firstStore); + + const secondStore = { get: vi.fn() } satisfies RunnerMatcherConfigStore; + createAwsSsmRunnerMatcherConfigStoreMock.mockReturnValue(secondStore); + resetRunnerMatcherConfigStore(); + + expect(getRunnerMatcherConfigStore()).toBe(secondStore); + expect(createAwsSsmRunnerMatcherConfigStoreMock).toHaveBeenCalledTimes(2); + }); +}); + +function setProvider(provider: string | undefined): void { + if (provider === undefined) { + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + } else { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + } +} + +function stubStore(): RunnerMatcherConfigStore { + const store = { get: vi.fn() } satisfies RunnerMatcherConfigStore; + createAwsSsmRunnerMatcherConfigStoreMock.mockReturnValue(store); + return store; +} diff --git a/lambdas/libs/storage-providers/runner-matcher-config.ts b/lambdas/libs/storage-providers/runner-matcher-config.ts new file mode 100644 index 0000000000..6d56d49754 --- /dev/null +++ b/lambdas/libs/storage-providers/runner-matcher-config.ts @@ -0,0 +1,23 @@ +import { createAwsSsmRunnerMatcherConfigStore } from './aws/ssm/runner-matcher-config-store'; +import type { RunnerMatcherConfigStore } from './core'; +import type {} from './environment'; +import { resolveRunnerConfigStorageProvider, type RunnerConfigStorageProvider } from './provider'; + +type RunnerMatcherConfigStoreFactory = () => RunnerMatcherConfigStore; + +const providerFactories = { + aws_ssm: createAwsSsmRunnerMatcherConfigStore, +} as const satisfies Record; + +let runnerMatcherConfigStore: RunnerMatcherConfigStore | undefined; + +export function getRunnerMatcherConfigStore(): RunnerMatcherConfigStore { + runnerMatcherConfigStore ??= + providerFactories[resolveRunnerConfigStorageProvider(process.env.RUNNER_CONFIG_STORAGE_PROVIDER)](); + return runnerMatcherConfigStore; +} + +// Test-only reset for cases that need to exercise first-use environment selection. +export function resetRunnerMatcherConfigStore(): void { + runnerMatcherConfigStore = undefined; +} diff --git a/lambdas/libs/storage-providers/vitest.config.ts b/lambdas/libs/storage-providers/vitest.config.ts index 20c739f253..f107f07411 100644 --- a/lambdas/libs/storage-providers/vitest.config.ts +++ b/lambdas/libs/storage-providers/vitest.config.ts @@ -13,6 +13,7 @@ export default mergeConfig(defaultConfig, { 'runner-config-consumer.ts', 'storage-providers.ts', 'provider.ts', + 'runner-matcher-config.ts', 'core/**/*.ts', 'aws/**/*.ts', ], diff --git a/lambdas/yarn.lock b/lambdas/yarn.lock index 0e7daca0c7..49dbdd1d66 100644 --- a/lambdas/yarn.lock +++ b/lambdas/yarn.lock @@ -243,6 +243,7 @@ __metadata: "@aws-github-runner/aws-powertools-util": "npm:*" "@aws-github-runner/aws-ssm-util": "npm:*" "@aws-github-runner/compute-providers": "npm:*" + "@aws-github-runner/storage-providers": "npm:*" "@aws-sdk/client-eventbridge": "npm:^3.1009.0" "@aws-sdk/client-sqs": "npm:^3.1009.0" "@middy/core": "npm:^6.4.5" From 10e40725d7e435fbb7ceec924c6b68dfdb67d1ce Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Wed, 19 Aug 2026 11:21:36 +0200 Subject: [PATCH 2/4] refactor(storage): extract webhook secret store --- lambdas/functions/webhook/package.json | 1 - .../webhook/src/ConfigLoader.test.ts | 62 +++++++------- lambdas/functions/webhook/src/ConfigLoader.ts | 25 +++--- lambdas/functions/webhook/src/lambda.test.ts | 15 +++- lambdas/functions/webhook/src/modules.d.ts | 1 - .../webhook/src/webhook/index.test.ts | 16 ++-- .../aws/ssm/environment.d.ts | 1 + .../ssm/github-webhook-secret-store.test.ts | 51 ++++++++++++ .../aws/ssm/github-webhook-secret-store.ts | 27 ++++++ lambdas/libs/storage-providers/core/index.ts | 3 + .../github-webhook-secret.test.ts | 83 +++++++++++++++++++ .../github-webhook-secret.ts | 23 +++++ lambdas/libs/storage-providers/index.ts | 2 + .../libs/storage-providers/vitest.config.ts | 1 + lambdas/yarn.lock | 1 - 15 files changed, 256 insertions(+), 56 deletions(-) create mode 100644 lambdas/libs/storage-providers/aws/ssm/github-webhook-secret-store.test.ts create mode 100644 lambdas/libs/storage-providers/aws/ssm/github-webhook-secret-store.ts create mode 100644 lambdas/libs/storage-providers/github-webhook-secret.test.ts create mode 100644 lambdas/libs/storage-providers/github-webhook-secret.ts diff --git a/lambdas/functions/webhook/package.json b/lambdas/functions/webhook/package.json index 83d9810d1f..c596db3493 100644 --- a/lambdas/functions/webhook/package.json +++ b/lambdas/functions/webhook/package.json @@ -29,7 +29,6 @@ }, "dependencies": { "@aws-github-runner/aws-powertools-util": "*", - "@aws-github-runner/aws-ssm-util": "*", "@aws-github-runner/compute-providers": "*", "@aws-github-runner/storage-providers": "*", "@aws-sdk/client-sqs": "^3.1009.0", diff --git a/lambdas/functions/webhook/src/ConfigLoader.test.ts b/lambdas/functions/webhook/src/ConfigLoader.test.ts index 7c7aa1dcf7..41bc66f13b 100644 --- a/lambdas/functions/webhook/src/ConfigLoader.test.ts +++ b/lambdas/functions/webhook/src/ConfigLoader.test.ts @@ -1,14 +1,20 @@ -import { getParameter } from '@aws-github-runner/aws-ssm-util'; -import { getRunnerMatcherConfigStore, type RunnerMatcherConfigStore } from '@aws-github-runner/storage-providers'; +import { + getGitHubWebhookSecretStore, + getRunnerMatcherConfigStore, + type GitHubWebhookSecretStore, + type RunnerMatcherConfigStore, +} from '@aws-github-runner/storage-providers'; import { ConfigWebhook, ConfigWebhookEventBridge, ConfigDispatcher } from './ConfigLoader'; import { logger } from '@aws-github-runner/aws-powertools-util'; import { RunnerMatcherConfig } from './sqs'; import { describe, it, expect, beforeEach, vi } from 'vitest'; -vi.mock('@aws-github-runner/aws-ssm-util'); vi.mock('@aws-github-runner/storage-providers'); +const githubWebhookSecretStore = { + get: vi.fn(), +} satisfies GitHubWebhookSecretStore; const runnerMatcherConfigStore = { get: vi.fn(), } satisfies RunnerMatcherConfigStore; @@ -20,6 +26,7 @@ describe('ConfigLoader Tests', () => { ConfigWebhookEventBridge.reset(); ConfigDispatcher.reset(); logger.setLogLevel('DEBUG'); + vi.mocked(getGitHubWebhookSecretStore).mockReturnValue(githubWebhookSecretStore); vi.mocked(getRunnerMatcherConfigStore).mockReturnValue(runnerMatcherConfigStore); // clear process.env @@ -31,7 +38,6 @@ describe('ConfigLoader Tests', () => { describe('Check base object', () => { function setupConfiguration(): void { process.env.EVENT_BUS_NAME = 'event-bus'; - process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; const matcherConfig = [ { id: '1', @@ -43,7 +49,7 @@ describe('ConfigLoader Tests', () => { }, ]; runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(matcherConfig)); - vi.mocked(getParameter).mockResolvedValue('secret'); + githubWebhookSecretStore.get.mockResolvedValue('secret'); } it('should return the same instance of ConfigWebhook (singleton)', async () => { @@ -52,7 +58,7 @@ describe('ConfigLoader Tests', () => { const config2 = await ConfigWebhook.load(); expect(config1).toBe(config2); - expect(getParameter).toHaveBeenCalledOnce(); + expect(githubWebhookSecretStore.get).toHaveBeenCalledOnce(); expect(runnerMatcherConfigStore.get).toHaveBeenCalledOnce(); }); @@ -62,7 +68,7 @@ describe('ConfigLoader Tests', () => { const config2 = await ConfigWebhookEventBridge.load(); expect(config1).toBe(config2); - expect(getParameter).toHaveBeenCalledTimes(1); + expect(githubWebhookSecretStore.get).toHaveBeenCalledOnce(); expect(runnerMatcherConfigStore.get).not.toHaveBeenCalled(); }); @@ -72,7 +78,7 @@ describe('ConfigLoader Tests', () => { const config2 = await ConfigDispatcher.load(); expect(config1).toBe(config2); - expect(getParameter).not.toHaveBeenCalled(); + expect(githubWebhookSecretStore.get).not.toHaveBeenCalled(); expect(runnerMatcherConfigStore.get).toHaveBeenCalledOnce(); }); @@ -96,7 +102,6 @@ describe('ConfigLoader Tests', () => { describe('ConfigWebhook', () => { it('should load config successfully', async () => { process.env.REPOSITORY_ALLOW_LIST = '["repo1", "repo2"]'; - process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; const matcherConfig = [ { id: '1', @@ -108,7 +113,7 @@ describe('ConfigLoader Tests', () => { }, ]; runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(matcherConfig)); - vi.mocked(getParameter).mockResolvedValue('secret'); + githubWebhookSecretStore.get.mockResolvedValue('secret'); const config: ConfigWebhook = await ConfigWebhook.load(); @@ -118,7 +123,6 @@ describe('ConfigLoader Tests', () => { }); it('should load config successfully', async () => { - process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; const matcherConfig = [ { id: '1', @@ -130,7 +134,7 @@ describe('ConfigLoader Tests', () => { }, ]; runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(matcherConfig)); - vi.mocked(getParameter).mockResolvedValue('secret'); + githubWebhookSecretStore.get.mockResolvedValue('secret'); const config: ConfigWebhook = await ConfigWebhook.load(); @@ -146,7 +150,7 @@ describe('ConfigLoader Tests', () => { 'Failed to load parameter for matcherConfig from path /path/to/matcher/config: Failed to load matcher config', ), ); - vi.mocked(getParameter).mockResolvedValue(''); + githubWebhookSecretStore.get.mockResolvedValue(''); await expect(ConfigWebhook.load()).rejects.toThrow( 'Failed to load config: Failed to load parameter for matcherConfig from path /path/to/matcher/config: Failed to load matcher config', @@ -154,14 +158,12 @@ describe('ConfigLoader Tests', () => { }); it('should load combined matcher config returned by the store', async () => { - process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; - const combinedMatcherConfig = [ { id: '1', arn: 'arn:aws:sqs:queue1', matcherConfig: { labelMatchers: [['a']], exactMatch: true } }, { id: '2', arn: 'arn:aws:sqs:queue2', matcherConfig: { labelMatchers: [['b']], exactMatch: true } }, ]; runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(combinedMatcherConfig)); - vi.mocked(getParameter).mockResolvedValue('secret'); + githubWebhookSecretStore.get.mockResolvedValue('secret'); const config: ConfigWebhook = await ConfigWebhook.load(); @@ -170,13 +172,12 @@ describe('ConfigLoader Tests', () => { }); it('should propagate an error from the matcher config store', async () => { - process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; runnerMatcherConfigStore.get.mockRejectedValue( new Error( "Failed to load/parse combined matcher config: Expected ',' or ']' after array element in JSON at position 196", ), ); - vi.mocked(getParameter).mockResolvedValue('secret'); + githubWebhookSecretStore.get.mockResolvedValue('secret'); await expect(ConfigWebhook.load()).rejects.toThrow( "Failed to load config: Failed to load/parse combined matcher config: Expected ',' or ']' after array element in JSON at position 196", @@ -188,14 +189,7 @@ describe('ConfigLoader Tests', () => { it('should load config successfully', async () => { process.env.ACCEPT_EVENTS = '["push", "pull_request"]'; process.env.EVENT_BUS_NAME = 'event-bus'; - process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; - - vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { - if (paramPath === '/path/to/webhook/secret') { - return 'secret'; - } - return ''; - }); + githubWebhookSecretStore.get.mockResolvedValue('secret'); const config: ConfigWebhookEventBridge = await ConfigWebhookEventBridge.load(); @@ -206,13 +200,23 @@ describe('ConfigLoader Tests', () => { }); it('should throw error if config loading fails', async () => { - vi.mocked(getParameter).mockImplementation(async (paramPath: string) => { - throw new Error(`Parameter ${paramPath} not found`); + githubWebhookSecretStore.get.mockRejectedValue(new Error('Webhook secret store is unavailable')); + + await expect(ConfigWebhookEventBridge.load()).rejects.toThrow( + 'Failed to load config: Environment variable for eventBusName is not set and no default value provided., Webhook secret store is unavailable', + ); + }); + + it('should report an error selecting the webhook secret store', async () => { + process.env.EVENT_BUS_NAME = 'event-bus'; + vi.mocked(getGitHubWebhookSecretStore).mockImplementationOnce(() => { + throw new Error("Unsupported runner config storage provider 'not-registered'"); }); await expect(ConfigWebhookEventBridge.load()).rejects.toThrow( - 'Failed to load config: Environment variable for eventBusName is not set and no default value provided., Failed to load parameter for webhookSecret from path undefined: Parameter undefined not found', + "Failed to load config: Unsupported runner config storage provider 'not-registered'", ); + expect(githubWebhookSecretStore.get).not.toHaveBeenCalled(); }); }); diff --git a/lambdas/functions/webhook/src/ConfigLoader.ts b/lambdas/functions/webhook/src/ConfigLoader.ts index 2cf261b849..e6d1d65004 100644 --- a/lambdas/functions/webhook/src/ConfigLoader.ts +++ b/lambdas/functions/webhook/src/ConfigLoader.ts @@ -1,10 +1,9 @@ -import { getParameter } from '@aws-github-runner/aws-ssm-util'; -import { getRunnerMatcherConfigStore } from '@aws-github-runner/storage-providers'; +import { getGitHubWebhookSecretStore, getRunnerMatcherConfigStore } from '@aws-github-runner/storage-providers'; import { RunnerMatcherConfig } from './sqs'; import { logger } from '@aws-github-runner/aws-powertools-util'; /** - * Base class for loading configuration from environment variables and SSM parameters. + * Base class for loading configuration from environment variables and configuration stores. * * @remarks * To avoid usages or checking values can be undefined we assume that configuration is @@ -55,16 +54,12 @@ abstract class BaseConfig { } } - protected async loadParameter(paramPath: string, propertyName: keyof this): Promise { - logger.debug(`Loading parameter for ${String(propertyName)} from path ${paramPath}`); - await getParameter(paramPath) - .then((value) => { - this.loadProperty(propertyName, value); - }) - .catch((error) => { - const errorMessage = `Failed to load parameter for ${String(propertyName)} from path ${paramPath}: ${(error as Error).message}`; - this.configLoadingErrors.push(errorMessage); - }); + protected async loadStoredProperty(propertyName: keyof this, getValue: () => Promise): Promise { + try { + this.loadProperty(propertyName, await getValue()); + } catch (error) { + this.configLoadingErrors.push((error as Error).message); + } } protected loadProperty(propertyName: keyof this, value: string) { @@ -117,7 +112,7 @@ export class ConfigWebhook extends MatcherAwareConfig { await Promise.all([ this.loadMatcherConfig(), - this.loadParameter(process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET, 'webhookSecret'), + this.loadStoredProperty('webhookSecret', () => getGitHubWebhookSecretStore().get()), ]); validateWebhookSecret(this); @@ -134,7 +129,7 @@ export class ConfigWebhookEventBridge extends BaseConfig { async loadConfig(): Promise { this.loadEnvVar(process.env.ACCEPT_EVENTS, 'allowedEvents', []); this.loadEnvVar(process.env.EVENT_BUS_NAME, 'eventBusName'); - await this.loadParameter(process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET, 'webhookSecret'); + await this.loadStoredProperty('webhookSecret', () => getGitHubWebhookSecretStore().get()); validateEventBusName(this); validateWebhookSecret(this); diff --git a/lambdas/functions/webhook/src/lambda.test.ts b/lambdas/functions/webhook/src/lambda.test.ts index 3bc67e42dd..b325c002f4 100644 --- a/lambdas/functions/webhook/src/lambda.test.ts +++ b/lambdas/functions/webhook/src/lambda.test.ts @@ -6,8 +6,12 @@ import { WorkflowJobEvent } from '@octokit/webhooks-types'; import { dispatchToRunners, eventBridgeWebhook, directWebhook } from './lambda'; import { publishForRunners, publishOnEventBridge } from './webhook'; import ValidationError from './ValidationError'; -import { getParameter } from '@aws-github-runner/aws-ssm-util'; -import { getRunnerMatcherConfigStore, type RunnerMatcherConfigStore } from '@aws-github-runner/storage-providers'; +import { + getGitHubWebhookSecretStore, + getRunnerMatcherConfigStore, + type GitHubWebhookSecretStore, + type RunnerMatcherConfigStore, +} from '@aws-github-runner/storage-providers'; import { dispatch } from './runners/dispatch'; import { EventWrapper } from './types'; import { describe, it, expect, beforeEach, vi } from 'vitest'; @@ -80,9 +84,11 @@ const context: Context = { vi.mock('./runners/dispatch'); vi.mock('./webhook'); -vi.mock('@aws-github-runner/aws-ssm-util'); vi.mock('@aws-github-runner/storage-providers'); +const githubWebhookSecretStore = { + get: vi.fn(), +} satisfies GitHubWebhookSecretStore; const runnerMatcherConfigStore = { get: vi.fn(), } satisfies RunnerMatcherConfigStore; @@ -92,7 +98,8 @@ describe('Test webhook lambda wrapper.', () => { vi.clearAllMocks(); // The handlers only need non-empty config values because their downstream // implementations are mocked in this wrapper test. - vi.mocked(getParameter).mockResolvedValue('["abc"]'); + vi.mocked(getGitHubWebhookSecretStore).mockReturnValue(githubWebhookSecretStore); + githubWebhookSecretStore.get.mockResolvedValue('["abc"]'); vi.mocked(getRunnerMatcherConfigStore).mockReturnValue(runnerMatcherConfigStore); runnerMatcherConfigStore.get.mockResolvedValue('["abc"]'); }); diff --git a/lambdas/functions/webhook/src/modules.d.ts b/lambdas/functions/webhook/src/modules.d.ts index 3b04a2a5be..05a81a12ab 100644 --- a/lambdas/functions/webhook/src/modules.d.ts +++ b/lambdas/functions/webhook/src/modules.d.ts @@ -2,7 +2,6 @@ declare namespace NodeJS { export interface ProcessEnv { ENVIRONMENT: string; EVENT_BUS_NAME: string; - PARAMETER_GITHUB_APP_WEBHOOK_SECRET: string; QUEUE_SELECTION_STRATEGY: string; REPOSITORY_ALLOW_LIST: string; RUNNER_LABELS: string; diff --git a/lambdas/functions/webhook/src/webhook/index.test.ts b/lambdas/functions/webhook/src/webhook/index.test.ts index 43345388f7..6d7a272309 100644 --- a/lambdas/functions/webhook/src/webhook/index.test.ts +++ b/lambdas/functions/webhook/src/webhook/index.test.ts @@ -1,6 +1,10 @@ import { Webhooks } from '@octokit/webhooks'; -import { getParameter } from '@aws-github-runner/aws-ssm-util'; -import { getRunnerMatcherConfigStore, type RunnerMatcherConfigStore } from '@aws-github-runner/storage-providers'; +import { + getGitHubWebhookSecretStore, + getRunnerMatcherConfigStore, + type GitHubWebhookSecretStore, + type RunnerMatcherConfigStore, +} from '@aws-github-runner/storage-providers'; import nock from 'nock'; @@ -16,10 +20,12 @@ import { describe, it, expect, beforeEach, vi } from 'vitest'; vi.mock('../sqs'); vi.mock('../eventbridge'); vi.mock('../runners/dispatch'); -vi.mock('@aws-github-runner/aws-ssm-util'); vi.mock('@aws-github-runner/storage-providers'); const GITHUB_APP_WEBHOOK_SECRET = 'TEST_SECRET'; +const githubWebhookSecretStore = { + get: vi.fn(), +} satisfies GitHubWebhookSecretStore; const runnerMatcherConfigStore = { get: vi.fn(), } satisfies RunnerMatcherConfigStore; @@ -290,7 +296,6 @@ describe('Check message size (checkBodySize)', () => { }); function mockConfigResponse() { - process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = '/path/to/webhook/secret'; const matcherConfig = [ { id: '1', @@ -301,7 +306,8 @@ function mockConfigResponse() { }, }, ]; + vi.mocked(getGitHubWebhookSecretStore).mockReturnValue(githubWebhookSecretStore); vi.mocked(getRunnerMatcherConfigStore).mockReturnValue(runnerMatcherConfigStore); + githubWebhookSecretStore.get.mockResolvedValue(GITHUB_APP_WEBHOOK_SECRET); runnerMatcherConfigStore.get.mockResolvedValue(JSON.stringify(matcherConfig)); - vi.mocked(getParameter).mockResolvedValue(GITHUB_APP_WEBHOOK_SECRET); } diff --git a/lambdas/libs/storage-providers/aws/ssm/environment.d.ts b/lambdas/libs/storage-providers/aws/ssm/environment.d.ts index 5ffa7b85bb..ab8b54e88a 100644 --- a/lambdas/libs/storage-providers/aws/ssm/environment.d.ts +++ b/lambdas/libs/storage-providers/aws/ssm/environment.d.ts @@ -3,6 +3,7 @@ export {}; declare global { namespace NodeJS { interface ProcessEnv { + PARAMETER_GITHUB_APP_WEBHOOK_SECRET?: string; PARAMETER_RUNNER_MATCHER_CONFIG_PATH?: string; SSM_PARAMETER_STORE_TAGS?: string; SSM_CONFIG_PATH?: string; diff --git a/lambdas/libs/storage-providers/aws/ssm/github-webhook-secret-store.test.ts b/lambdas/libs/storage-providers/aws/ssm/github-webhook-secret-store.test.ts new file mode 100644 index 0000000000..7384c769bc --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm/github-webhook-secret-store.test.ts @@ -0,0 +1,51 @@ +import { getParameter } from '@aws-github-runner/aws-ssm-util'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAwsSsmGitHubWebhookSecretStore } from './github-webhook-secret-store'; + +vi.mock('@aws-github-runner/aws-ssm-util', () => ({ + getParameter: vi.fn(), +})); + +const getParameterMock = vi.mocked(getParameter); +const cleanEnv = process.env; +const webhookSecretParameter = '/actions-runner/test/webhook_secret'; + +describe('aws_ssm GitHub webhook secret store', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env = { ...cleanEnv }; + process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = webhookSecretParameter; + }); + + it('loads the webhook secret parameter', async () => { + getParameterMock.mockResolvedValue('fake-webhook-secret'); + const store = createAwsSsmGitHubWebhookSecretStore(); + + await expect(store.get()).resolves.toBe('fake-webhook-secret'); + expect(getParameterMock).toHaveBeenCalledOnce(); + expect(getParameterMock).toHaveBeenCalledWith(webhookSecretParameter); + }); + + it('wraps a parameter read failure with the legacy error message', async () => { + getParameterMock.mockRejectedValue(new Error('access denied')); + const store = createAwsSsmGitHubWebhookSecretStore(); + + await expect(store.get()).rejects.toThrow( + `Failed to load parameter for webhookSecret from path ${webhookSecretParameter}: access denied`, + ); + }); + + it.each([undefined, '', ' '])('requires a webhook secret parameter path for input %j', (parameterPath) => { + if (parameterPath === undefined) { + delete process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET; + } else { + process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET = parameterPath; + } + + expect(() => createAwsSsmGitHubWebhookSecretStore()).toThrow( + 'Environment variable PARAMETER_GITHUB_APP_WEBHOOK_SECRET is not set', + ); + expect(getParameterMock).not.toHaveBeenCalled(); + }); +}); diff --git a/lambdas/libs/storage-providers/aws/ssm/github-webhook-secret-store.ts b/lambdas/libs/storage-providers/aws/ssm/github-webhook-secret-store.ts new file mode 100644 index 0000000000..ce35e1f532 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm/github-webhook-secret-store.ts @@ -0,0 +1,27 @@ +import { getParameter } from '@aws-github-runner/aws-ssm-util'; + +import type { GitHubWebhookSecretStore } from '../../core'; +import type {} from './environment'; + +export function createAwsSsmGitHubWebhookSecretStore(): GitHubWebhookSecretStore { + const parameterPath = process.env.PARAMETER_GITHUB_APP_WEBHOOK_SECRET; + if (!parameterPath || parameterPath.trim() === '') { + throw new Error('Environment variable PARAMETER_GITHUB_APP_WEBHOOK_SECRET is not set'); + } + + return new AwsSsmGitHubWebhookSecretStore(parameterPath); +} + +class AwsSsmGitHubWebhookSecretStore implements GitHubWebhookSecretStore { + constructor(private readonly parameterPath: string) {} + + async get(): Promise { + try { + return await getParameter(this.parameterPath); + } catch (error) { + throw new Error( + `Failed to load parameter for webhookSecret from path ${this.parameterPath}: ${(error as Error).message}`, + ); + } + } +} diff --git a/lambdas/libs/storage-providers/core/index.ts b/lambdas/libs/storage-providers/core/index.ts index 3d9fc43fe8..e6060eae2a 100644 --- a/lambdas/libs/storage-providers/core/index.ts +++ b/lambdas/libs/storage-providers/core/index.ts @@ -1,3 +1,6 @@ +export interface GitHubWebhookSecretStore { + get(): Promise; +} export interface RunnerConfigMetadata { key: string; value: string; diff --git a/lambdas/libs/storage-providers/github-webhook-secret.test.ts b/lambdas/libs/storage-providers/github-webhook-secret.test.ts new file mode 100644 index 0000000000..5888e735b0 --- /dev/null +++ b/lambdas/libs/storage-providers/github-webhook-secret.test.ts @@ -0,0 +1,83 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAwsSsmGitHubWebhookSecretStore } from './aws/ssm/github-webhook-secret-store'; +import type { GitHubWebhookSecretStore } from './core'; +import { getGitHubWebhookSecretStore, resetGitHubWebhookSecretStore } from './github-webhook-secret'; + +vi.mock('./aws/ssm/github-webhook-secret-store', () => ({ + createAwsSsmGitHubWebhookSecretStore: vi.fn(), +})); + +const createAwsSsmGitHubWebhookSecretStoreMock = vi.mocked(createAwsSsmGitHubWebhookSecretStore); +const cleanEnv = process.env; + +describe('GitHub webhook secret store selection', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env = { ...cleanEnv }; + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + resetGitHubWebhookSecretStore(); + }); + + it.each([undefined, '', ' '])('uses aws_ssm for default selector input %j', (provider) => { + setProvider(provider); + const store = stubStore(); + + expect(getGitHubWebhookSecretStore()).toBe(store); + expect(createAwsSsmGitHubWebhookSecretStoreMock).toHaveBeenCalledOnce(); + }); + + it.each(['aws_ssm', ' AWS_SSM '])('uses aws_ssm for explicit selector input %j', (provider) => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + const store = stubStore(); + + expect(getGitHubWebhookSecretStore()).toBe(store); + expect(createAwsSsmGitHubWebhookSecretStoreMock).toHaveBeenCalledOnce(); + }); + + it('rejects an unsupported provider on first use', () => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = 'not-registered'; + + expect(() => getGitHubWebhookSecretStore()).toThrow("Unsupported runner config storage provider 'not-registered'"); + expect(createAwsSsmGitHubWebhookSecretStoreMock).not.toHaveBeenCalled(); + }); + + it('selects lazily and caches the created store', () => { + const store = stubStore(); + + expect(createAwsSsmGitHubWebhookSecretStoreMock).not.toHaveBeenCalled(); + const first = getGitHubWebhookSecretStore(); + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = 'not-registered'; + const second = getGitHubWebhookSecretStore(); + + expect(first).toBe(store); + expect(second).toBe(store); + expect(createAwsSsmGitHubWebhookSecretStoreMock).toHaveBeenCalledOnce(); + }); + + it('selects again after the test reset', () => { + const firstStore = stubStore(); + expect(getGitHubWebhookSecretStore()).toBe(firstStore); + + const secondStore = { get: vi.fn() } satisfies GitHubWebhookSecretStore; + createAwsSsmGitHubWebhookSecretStoreMock.mockReturnValue(secondStore); + resetGitHubWebhookSecretStore(); + + expect(getGitHubWebhookSecretStore()).toBe(secondStore); + expect(createAwsSsmGitHubWebhookSecretStoreMock).toHaveBeenCalledTimes(2); + }); +}); + +function setProvider(provider: string | undefined): void { + if (provider === undefined) { + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + } else { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + } +} + +function stubStore(): GitHubWebhookSecretStore { + const store = { get: vi.fn() } satisfies GitHubWebhookSecretStore; + createAwsSsmGitHubWebhookSecretStoreMock.mockReturnValue(store); + return store; +} diff --git a/lambdas/libs/storage-providers/github-webhook-secret.ts b/lambdas/libs/storage-providers/github-webhook-secret.ts new file mode 100644 index 0000000000..f13df08718 --- /dev/null +++ b/lambdas/libs/storage-providers/github-webhook-secret.ts @@ -0,0 +1,23 @@ +import { createAwsSsmGitHubWebhookSecretStore } from './aws/ssm/github-webhook-secret-store'; +import type { GitHubWebhookSecretStore } from './core'; +import type {} from './environment'; +import { resolveRunnerConfigStorageProvider, type RunnerConfigStorageProvider } from './provider'; + +type GitHubWebhookSecretStoreFactory = () => GitHubWebhookSecretStore; + +const providerFactories = { + aws_ssm: createAwsSsmGitHubWebhookSecretStore, +} as const satisfies Record; + +let githubWebhookSecretStore: GitHubWebhookSecretStore | undefined; + +export function getGitHubWebhookSecretStore(): GitHubWebhookSecretStore { + githubWebhookSecretStore ??= + providerFactories[resolveRunnerConfigStorageProvider(process.env.RUNNER_CONFIG_STORAGE_PROVIDER)](); + return githubWebhookSecretStore; +} + +// Test-only reset for cases that need to exercise first-use environment selection. +export function resetGitHubWebhookSecretStore(): void { + githubWebhookSecretStore = undefined; +} diff --git a/lambdas/libs/storage-providers/index.ts b/lambdas/libs/storage-providers/index.ts index 026b79d48a..fcd4bd9338 100644 --- a/lambdas/libs/storage-providers/index.ts +++ b/lambdas/libs/storage-providers/index.ts @@ -1,6 +1,7 @@ export type { GitHubAppCredential, GitHubAppCredentialsStore, + GitHubWebhookSecretStore, RunnerConfigConsumer, RunnerConfigConsumeOptions, RunnerConfigHousekeeper, @@ -21,4 +22,5 @@ export { export type { RunnerConfigStorageProvider } from './provider'; export { createCommonStorage, createStorageProviders } from './storage-providers'; export type { StorageProviders, RunnerConfigStorage, CommonStorage } from './core'; +export { getGitHubWebhookSecretStore, resetGitHubWebhookSecretStore } from './github-webhook-secret'; export { getRunnerMatcherConfigStore, resetRunnerMatcherConfigStore } from './runner-matcher-config'; diff --git a/lambdas/libs/storage-providers/vitest.config.ts b/lambdas/libs/storage-providers/vitest.config.ts index f107f07411..a009dab1bf 100644 --- a/lambdas/libs/storage-providers/vitest.config.ts +++ b/lambdas/libs/storage-providers/vitest.config.ts @@ -13,6 +13,7 @@ export default mergeConfig(defaultConfig, { 'runner-config-consumer.ts', 'storage-providers.ts', 'provider.ts', + 'github-webhook-secret.ts', 'runner-matcher-config.ts', 'core/**/*.ts', 'aws/**/*.ts', diff --git a/lambdas/yarn.lock b/lambdas/yarn.lock index 49dbdd1d66..17e99e638d 100644 --- a/lambdas/yarn.lock +++ b/lambdas/yarn.lock @@ -241,7 +241,6 @@ __metadata: resolution: "@aws-github-runner/webhook@workspace:functions/webhook" dependencies: "@aws-github-runner/aws-powertools-util": "npm:*" - "@aws-github-runner/aws-ssm-util": "npm:*" "@aws-github-runner/compute-providers": "npm:*" "@aws-github-runner/storage-providers": "npm:*" "@aws-sdk/client-eventbridge": "npm:^3.1009.0" From 1d9fa0e262371a749aaa7248be78d9b59758f703 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 8 Sep 2026 23:22:32 +0200 Subject: [PATCH 3/4] feat(storage): add shared DynamoDB runner storage --- .../src/pool/pool-contract.test.ts | 7 + .../functions/control-plane/src/pool/pool.ts | 69 ++- .../src/scale-runners/github-runner.ts | 195 +++++- .../src/scale-runners/job-retry.test.ts | 7 + .../scale-runners/scale-down-contract.test.ts | 4 + .../src/scale-runners/scale-down.test.ts | 353 +++++++++++ .../src/scale-runners/scale-down.ts | 388 ++++++++++-- .../scale-runners/scale-up-contract.test.ts | 9 + .../src/scale-runners/scale-up.test.ts | 4 +- .../src/scale-runners/scale-up.ts | 39 +- .../compute-provider-contracts/scale-up.ts | 4 +- .../ec2/src/control-plane/runner-creation.ts | 3 + .../ec2/src/control-plane/scale-up.test.ts | 2 +- .../aws/ec2/src/control-plane/scale-up.ts | 4 +- .../aws/ec2/src/environment.d.ts | 1 + lambdas/libs/compute-providers/core/index.ts | 9 +- .../storage-providers/aws/dynamodb/client.ts | 21 + .../aws/dynamodb/durable-config.ts | 37 ++ .../aws/dynamodb/environment.d.ts | 13 + .../aws/dynamodb/environment.ts | 29 + .../github-app-credentials-store.test.ts | 103 ++++ .../dynamodb/github-app-credentials-store.ts | 89 +++ .../github-webhook-secret-store.test.ts | 74 +++ .../dynamodb/github-webhook-secret-store.ts | 21 + .../aws/dynamodb/keys.test.ts | 50 ++ .../storage-providers/aws/dynamodb/keys.ts | 34 ++ .../aws/dynamodb/runner-config-store.test.ts | 148 +++++ .../aws/dynamodb/runner-config-store.ts | 66 +++ .../dynamodb/runner-group-cache-store.test.ts | 108 ++++ .../aws/dynamodb/runner-group-cache-store.ts | 80 +++ .../runner-matcher-config-store.test.ts | 95 +++ .../dynamodb/runner-matcher-config-store.ts | 27 + .../aws/dynamodb/runner-state-store.test.ts | 451 ++++++++++++++ .../aws/dynamodb/runner-state-store.ts | 556 ++++++++++++++++++ lambdas/libs/storage-providers/core/index.ts | 51 ++ .../github-app-credentials.test.ts | 97 +++ .../github-app-credentials.ts | 25 + .../github-webhook-secret.test.ts | 56 +- .../github-webhook-secret.ts | 2 + lambdas/libs/storage-providers/index.ts | 12 + lambdas/libs/storage-providers/package.json | 1 + lambdas/libs/storage-providers/provider.ts | 6 +- .../storage-providers/runner-config.test.ts | 108 ++++ .../libs/storage-providers/runner-config.ts | 25 + .../runner-group-cache.test.ts | 106 ++++ .../storage-providers/runner-group-cache.ts | 25 + .../runner-matcher-config.test.ts | 23 + .../runner-matcher-config.ts | 2 + .../storage-providers/runner-state.test.ts | 89 +++ .../libs/storage-providers/runner-state.ts | 25 + .../libs/storage-providers/vitest.config.ts | 1 + lambdas/yarn.lock | 342 +++++++++++ 52 files changed, 3998 insertions(+), 98 deletions(-) create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/client.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/durable-config.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/environment.d.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/environment.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/github-app-credentials-store.test.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/github-app-credentials-store.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/github-webhook-secret-store.test.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/github-webhook-secret-store.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/keys.test.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/keys.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/runner-config-store.test.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/runner-config-store.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/runner-group-cache-store.test.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/runner-group-cache-store.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/runner-matcher-config-store.test.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/runner-matcher-config-store.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/runner-state-store.test.ts create mode 100644 lambdas/libs/storage-providers/aws/dynamodb/runner-state-store.ts create mode 100644 lambdas/libs/storage-providers/github-app-credentials.test.ts create mode 100644 lambdas/libs/storage-providers/github-app-credentials.ts create mode 100644 lambdas/libs/storage-providers/runner-config.test.ts create mode 100644 lambdas/libs/storage-providers/runner-config.ts create mode 100644 lambdas/libs/storage-providers/runner-group-cache.test.ts create mode 100644 lambdas/libs/storage-providers/runner-group-cache.ts create mode 100644 lambdas/libs/storage-providers/runner-state.test.ts create mode 100644 lambdas/libs/storage-providers/runner-state.ts diff --git a/lambdas/functions/control-plane/src/pool/pool-contract.test.ts b/lambdas/functions/control-plane/src/pool/pool-contract.test.ts index 1a208edde7..4f9628fd05 100644 --- a/lambdas/functions/control-plane/src/pool/pool-contract.test.ts +++ b/lambdas/functions/control-plane/src/pool/pool-contract.test.ts @@ -17,6 +17,13 @@ vi.mock('../github/auth', () => ({ getStoredInstallationId: vi.fn().mockResolvedValue(undefined), })); +vi.mock('@aws-github-runner/storage-providers', () => ({ + createStorageProviders: vi.fn().mockReturnValue({ + githubAppCredentials: { get: vi.fn() }, + }), + getRunnerStateStore: vi.fn().mockReturnValue(undefined), +})); + vi.mock('../scale-runners/github-runner', () => ({ createStartRunnerConfig: vi.fn(), getGitHubEnterpriseApiUrl: vi.fn(), diff --git a/lambdas/functions/control-plane/src/pool/pool.ts b/lambdas/functions/control-plane/src/pool/pool.ts index 9f91c6cbcd..60c7bc2646 100644 --- a/lambdas/functions/control-plane/src/pool/pool.ts +++ b/lambdas/functions/control-plane/src/pool/pool.ts @@ -1,7 +1,12 @@ import { Octokit } from '@octokit/rest'; import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; import { resolveComputeProviderType } from '@aws-github-runner/compute-providers/provider-types'; -import { createStorageProviders, type StorageProviders } from '@aws-github-runner/storage-providers'; +import { + createStorageProviders, + getRunnerStateStore, + type RunnerStateRecord, + type StorageProviders, +} from '@aws-github-runner/storage-providers'; import yn from 'yn'; import { @@ -60,26 +65,43 @@ export async function adjust(event: PoolEvent): Promise { runnerNamePrefix, ); - // Look up the managed provider runners, but running does not mean idle. - const poolRunners = await computeProvider.listRunners({ + const runnerStateStore = getRunnerStateStore(); + let currentRunnerCount: number; + let numberOfRunnersInPool: number; + const providerRunners = await computeProvider.listRunners({ environment, runnerOwner, runnerType: 'Org', }); - - const numberOfRunnersInPool = computeProvider.countAvailableRunners(poolRunners, runnerStatusses, includeBusyRunners); + if (runnerStateStore) { + const runnerStates = (await runnerStateStore.list({ computeProvider: computeProvider.type })).filter( + (record) => record.runnerOwner === runnerOwner && record.runnerType === 'Org', + ); + const storedComputeResourceIds = new Set(runnerStates.map((record) => record.computeResourceId)); + const untrackedProviderRunners = providerRunners.filter((runner) => !storedComputeResourceIds.has(runner.id)); + // Inventory is canonical for tracked resources. Provider discovery contributes + // only untracked resources so a launch-before-state crash cannot over-provision. + currentRunnerCount = runnerStates.length + untrackedProviderRunners.length; + numberOfRunnersInPool = + countAvailableStoredRunners(runnerStates, runnerStatusses, includeBusyRunners) + + computeProvider.countAvailableRunners(untrackedProviderRunners, runnerStatusses, includeBusyRunners); + } else { + // Look up the managed provider runners, but running does not mean idle. + currentRunnerCount = providerRunners.length; + numberOfRunnersInPool = computeProvider.countAvailableRunners(providerRunners, runnerStatusses, includeBusyRunners); + } let topUp = event.poolSize - numberOfRunnersInPool; // The pool must never push the total number of runners (busy + idle) past the configured maximum. - // poolRunners contains every running runner for this type, so its length is the current total and no - // extra API call is needed. Without this clamp the pool keeps topping up against idle-only counts and - // can overshoot runners_maximum_count, while the scale-up lambda correctly refuses to launch. + // currentRunnerCount includes both canonical inventory and untracked provider recovery records. Without + // this clamp the pool keeps topping up against idle-only counts and can overshoot runners_maximum_count, + // while the scale-up lambda correctly refuses to launch. if (maximumRunners !== -1 && topUp > 0) { - const headroom = maximumRunners - poolRunners.length; + const headroom = maximumRunners - currentRunnerCount; if (topUp > headroom) { logger.info( `Capping pool top-up from ${topUp} to ${Math.max(headroom, 0)} to respect the maximum of ` + - `${maximumRunners} runners (currently ${poolRunners.length} running).`, + `${maximumRunners} runners (currently ${currentRunnerCount} running).`, ); topUp = headroom; } @@ -129,6 +151,33 @@ async function getInstallationId( ).data.id; } +function countAvailableStoredRunners( + runnerStates: RunnerStateRecord[], + runnerStatuses: Map, + includeBusyRunners: boolean, +): number { + let available = 0; + for (const runner of runnerStates) { + if (runner.state === 'orphan' || runner.state === 'terminating') { + continue; + } + + const status = runnerStatuses.get(runner.computeResourceId); + if ((status?.busy === false || includeBusyRunners) && status?.status === 'online') { + available++; + } else if (status === undefined && !runnerBootTimeExceeded(runner.createdAt)) { + available++; + } + } + return available; +} + +function runnerBootTimeExceeded(createdAt: string): boolean { + const bootTimeMinutes = Number(process.env.RUNNER_BOOT_TIME_IN_MINUTES); + const launchTime = new Date(createdAt).getTime(); + return launchTime + bootTimeMinutes * 60_000 < Date.now(); +} + async function getGitHubRegisteredRunnnerStatusses( ghClient: Octokit, runnerOwner: string, diff --git a/lambdas/functions/control-plane/src/scale-runners/github-runner.ts b/lambdas/functions/control-plane/src/scale-runners/github-runner.ts index b344012149..b697ba5884 100644 --- a/lambdas/functions/control-plane/src/scale-runners/github-runner.ts +++ b/lambdas/functions/control-plane/src/scale-runners/github-runner.ts @@ -1,11 +1,16 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; import { createStorageProviders, + getRunnerConfigStore, + getRunnerStateStore, type RunnerConfigMetadata, + type RunnerConfigRecord, type RunnerConfigStore, type GitHubAppCredentialsStore, type RunnerGroupCacheStore, + type RunnerStateStore, } from '@aws-github-runner/storage-providers'; +import type { ComputeProviderType } from '@aws-github-runner/compute-providers/provider-types'; import { Octokit } from '@octokit/rest'; import { getStoredInstallationId } from '../github/auth'; @@ -22,6 +27,8 @@ export interface GitHubRunnerMetadata { export interface StartRunnerConfigOptions { runnerConfigStore?: RunnerConfigStore; runnerGroupCacheStore?: RunnerGroupCacheStore; + computeProvider?: ComputeProviderType; + getRunnerConfigAccessScope?: (runnerId: string) => string | undefined; getRunnerConfigMetadata?: (runnerId: string) => RunnerConfigMetadata[]; onJitConfigCreated?: (runnerId: string, metadata: GitHubRunnerMetadata) => Promise; } @@ -207,11 +214,22 @@ export async function createStartRunnerConfig( ghClient: Octokit, options: StartRunnerConfigOptions = {}, ): Promise { - const runnerConfigStore = options.runnerConfigStore ?? createStorageProviders().runnerConfig; + const runnerConfigStore = options.runnerConfigStore ?? getRunnerConfigStore(); + const runnerStateStore = getRunnerStateStore(); + if (runnerStateStore && options.computeProvider === undefined) { + throw new Error('A compute provider is required when runner state storage is enabled'); + } if (githubRunnerConfig.enableJitConfig && githubRunnerConfig.ephemeral) { - return await createJitConfig(githubRunnerConfig, runnerIds, ghClient, runnerConfigStore, options); + return await createJitConfig(githubRunnerConfig, runnerIds, ghClient, runnerConfigStore, runnerStateStore, options); } else { - return await createRegistrationTokenConfig(githubRunnerConfig, runnerIds, ghClient, runnerConfigStore, options); + return await createRegistrationTokenConfig( + githubRunnerConfig, + runnerIds, + ghClient, + runnerConfigStore, + runnerStateStore, + options, + ); } } @@ -226,13 +244,15 @@ function addDelay(runnerIds: string[], runnerConfigStore: RunnerConfigStore) { /** * Creates registration token configuration for non-ephemeral runners. * - * @returns Empty array (this configuration method does not have failure cases) + * @returns Runner IDs whose durable configuration failed. The legacy SSM path + * still throws on the first failure so existing all-or-nothing cleanup is preserved. */ async function createRegistrationTokenConfig( githubRunnerConfig: CreateGitHubRunnerConfig, runnerIds: string[], ghClient: Octokit, runnerConfigStore: RunnerConfigStore, + runnerStateStore: RunnerStateStore | undefined, options: StartRunnerConfigOptions, ): Promise { const { isDelay, delay, delayMilliseconds } = addDelay(runnerIds, runnerConfigStore); @@ -243,18 +263,53 @@ async function createRegistrationTokenConfig( runner_service_config: removeTokenFromLogging(runnerServiceConfig), }); + if (!runnerStateStore) { + for (const runnerId of runnerIds) { + const metadata = options.getRunnerConfigMetadata?.(runnerId); + await runnerConfigStore.create(createRunnerConfigRecord(runnerId, runnerServiceConfig.join(' '), options), { + metadata, + }); + if (isDelay) { + // Delay to stay within the selected store's maximum write throughput. + await delay(delayMilliseconds); + } + } + return []; + } + + const failedRunnerIds: string[] = []; for (const runnerId of runnerIds) { - await runnerConfigStore.create( - { runnerId, value: runnerServiceConfig.join(' ') }, - { metadata: options.getRunnerConfigMetadata?.(runnerId) }, - ); - if (isDelay) { - // Delay to stay within the selected store's maximum write throughput. - await delay(delayMilliseconds); + try { + const metadata = options.getRunnerConfigMetadata?.(runnerId); + await createProvisioningRunnerState(runnerStateStore, githubRunnerConfig, runnerId, options, metadata); + await runnerConfigStore.create(createRunnerConfigRecord(runnerId, runnerServiceConfig.join(' '), options), { + metadata, + }); + await runnerStateStore.activate(runnerId, { metadata }); + if (isDelay) { + // Delay to stay within the selected store's maximum write throughput. + await delay(delayMilliseconds); + } + } catch (error) { + failedRunnerIds.push(runnerId); + logger.warn('Failed to configure runner, continuing with remaining instances', { + instance: runnerId, + error: error instanceof Error ? error.message : String(error), + retryable: true, + }); } } - return []; + if (failedRunnerIds.length > 0) { + logger.error('Failed to configure some runner instances', { + failedInstances: failedRunnerIds, + totalInstances: runnerIds.length, + successfulInstances: runnerIds.length - failedRunnerIds.length, + retryable: true, + }); + } + + return failedRunnerIds; } /** @@ -268,6 +323,7 @@ async function createJitConfig( runnerIds: string[], ghClient: Octokit, runnerConfigStore: RunnerConfigStore, + runnerStateStore: RunnerStateStore | undefined, options: StartRunnerConfigOptions, ): Promise { const runnerGroupId = await getRunnerGroupId(githubRunnerConfig, ghClient, options.runnerGroupCacheStore); @@ -285,6 +341,17 @@ async function createJitConfig( runnerGroupId: runnerGroupId, runnerLabels: runnerLabels, }; + // Durable inventory needs metadata before JIT creation so failures leave a + // complete provisioning record. The legacy path retains its original timing. + let metadata = runnerStateStore ? options.getRunnerConfigMetadata?.(runnerId) : undefined; + await createProvisioningRunnerState( + runnerStateStore, + githubRunnerConfig, + runnerId, + options, + metadata, + ephemeralRunnerConfig, + ); logger.debug(`Runner name: ${ephemeralRunnerConfig.runnerName}`); const runnerConfig = githubRunnerConfig.runnerType === 'Org' @@ -304,18 +371,47 @@ async function createJitConfig( metricGitHubAppRateLimit(runnerConfig.headers, githubRunnerConfig.appIndex); - await options.onJitConfigCreated?.(runnerId, { + const githubRunnerMetadata = { githubRunnerId: runnerConfig.data.runner.id.toString(), runnerLabels, - }); + }; + if (runnerStateStore) { + try { + await runnerStateStore.recordGitHubIdentity(runnerId, { + ...githubRunnerMetadata, + runnerName: ephemeralRunnerConfig.runnerName, + metadata, + }); + } catch (error) { + await deregisterJitRunnerAfterIdentityWriteFailure( + githubRunnerConfig, + ghClient, + runnerConfig.data.runner.id, + runnerId, + ); + throw error; + } + } + await options.onJitConfigCreated?.(runnerId, githubRunnerMetadata); logger.debug('Runner JIT config for ephemeral runner generated.', { instance: runnerId, }); + if (!runnerStateStore) { + metadata = options.getRunnerConfigMetadata?.(runnerId); + } await runnerConfigStore.create( - { runnerId, value: runnerConfig.data.encoded_jit_config }, - { metadata: options.getRunnerConfigMetadata?.(runnerId) }, + createRunnerConfigRecord(runnerId, runnerConfig.data.encoded_jit_config, options), + { + metadata, + }, ); + await runnerStateStore?.activate(runnerId, { + githubRunnerId: githubRunnerMetadata.githubRunnerId, + runnerLabels, + runnerName: ephemeralRunnerConfig.runnerName, + metadata, + }); if (isDelay) { // Delay to stay within the selected store's maximum write throughput. await delay(delayMilliseconds); @@ -342,6 +438,73 @@ async function createJitConfig( return failedRunnerIds; } +async function deregisterJitRunnerAfterIdentityWriteFailure( + githubRunnerConfig: CreateGitHubRunnerConfig, + ghClient: Octokit, + githubRunnerId: number, + runnerId: string, +): Promise { + try { + if (githubRunnerConfig.runnerType === 'Org') { + await ghClient.actions.deleteSelfHostedRunnerFromOrg({ + org: githubRunnerConfig.runnerOwner, + runner_id: githubRunnerId, + }); + } else { + const [owner, repo] = githubRunnerConfig.runnerOwner.split('/'); + await ghClient.actions.deleteSelfHostedRunnerFromRepo({ owner, repo, runner_id: githubRunnerId }); + } + logger.info('De-registered JIT GitHub runner after its inventory identity could not be stored', { + instance: runnerId, + githubRunnerId, + }); + } catch (cleanupError) { + logger.error('Failed to de-register JIT GitHub runner after its inventory identity could not be stored', { + instance: runnerId, + githubRunnerId, + error: cleanupError instanceof Error ? cleanupError.message : String(cleanupError), + retryable: true, + }); + } +} + +function createRunnerConfigRecord( + runnerId: string, + value: string, + options: StartRunnerConfigOptions, +): RunnerConfigRecord { + const accessScope = options.getRunnerConfigAccessScope?.(runnerId); + return { + runnerId, + value, + ...(accessScope === undefined ? {} : { accessScope }), + }; +} + +async function createProvisioningRunnerState( + runnerStateStore: RunnerStateStore | undefined, + githubRunnerConfig: CreateGitHubRunnerConfig, + runnerId: string, + options: StartRunnerConfigOptions, + metadata: RunnerConfigMetadata[] | undefined, + jitConfig?: EphemeralRunnerConfig, +): Promise { + if (!runnerStateStore) { + return; + } + + await runnerStateStore.create({ + runnerId, + computeProvider: options.computeProvider!, + computeResourceId: runnerId, + runnerOwner: githubRunnerConfig.runnerOwner, + runnerType: githubRunnerConfig.runnerType, + runnerName: jitConfig?.runnerName, + runnerLabels: jitConfig?.runnerLabels, + metadata, + }); +} + export function getGitHubEnterpriseApiUrl() { const ghesBaseUrl = process.env.GHES_URL; let ghesApiUrl = ''; diff --git a/lambdas/functions/control-plane/src/scale-runners/job-retry.test.ts b/lambdas/functions/control-plane/src/scale-runners/job-retry.test.ts index c4ff1e5d76..dd1cd503ea 100644 --- a/lambdas/functions/control-plane/src/scale-runners/job-retry.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/job-retry.test.ts @@ -12,6 +12,13 @@ vi.mock('../aws/sqs', async () => ({ publishMessage: vi.fn(), })); +vi.mock('@aws-github-runner/storage-providers', () => ({ + getGitHubAppCredentialsStore: vi.fn(), + getRunnerConfigStore: vi.fn().mockReturnValue({ houseKeeper: vi.fn() }), + getRunnerGroupCacheStore: vi.fn(), + getRunnerStateStore: vi.fn().mockReturnValue(undefined), +})); + vi.mock('@aws-github-runner/aws-powertools-util', async () => { // This is a workaround for TypeScript's type checking // Use vi.importActual with a type assertion to avoid spread operator type error diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down-contract.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down-contract.test.ts index afd25211da..fe51dc3a9c 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down-contract.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down-contract.test.ts @@ -7,6 +7,10 @@ import { controlPlaneProviderRegistry } from '../control-plane-providers'; import { scaleDown } from './scale-down'; import type { ScaleDownComputeProvider } from './types'; +vi.mock('@aws-github-runner/storage-providers', () => ({ + getRunnerStateStore: vi.fn().mockReturnValue(undefined), +})); + const mockedResolveCapability = vi.spyOn(controlPlaneProviderRegistry, 'capability'); const cleanEnv = process.env; diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts index 743068e00c..b5de6e1ec7 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.test.ts @@ -1,5 +1,10 @@ import type { Octokit } from '@octokit/rest'; import { RequestError } from '@octokit/request-error'; +import { + getRunnerStateStore, + type RunnerStateRecord, + type RunnerStateStore, +} from '@aws-github-runner/storage-providers'; import moment from 'moment'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { defaultComputeProvider } from '@aws-github-runner/compute-providers/provider-types'; @@ -17,6 +22,10 @@ vi.mock('../github/auth', () => ({ getStoredInstallationId: vi.fn().mockResolvedValue(undefined), })); +vi.mock('@aws-github-runner/storage-providers', () => ({ + getRunnerStateStore: vi.fn(), +})); + const mockOctokit = { apps: { getOrgInstallation: vi.fn(), @@ -50,6 +59,18 @@ const mockBootTimeExceeded = vi.mocked(mockComputeProvider.bootTimeExceeded); const mockMarkOrphan = vi.mocked(mockComputeProvider.markOrphan); const mockUnmarkOrphan = vi.mocked(mockComputeProvider.unmarkOrphan); const mockTerminateRunners = vi.mocked(mockComputeProvider.terminate); +const mockGetRunnerStateStore = vi.mocked(getRunnerStateStore); +const mockRunnerStateStore: RunnerStateStore = { + create: vi.fn(), + recordGitHubIdentity: vi.fn(), + activate: vi.fn(), + list: vi.fn(), + markOrphan: vi.fn(), + unmarkOrphan: vi.fn(), + beginTermination: vi.fn(), + cancelTermination: vi.fn(), + delete: vi.fn(), +}; const cleanEnv = process.env; @@ -191,6 +212,14 @@ describe('Scale down runners', () => { mockMarkOrphan.mockResolvedValue(); mockUnmarkOrphan.mockResolvedValue(); mockTerminateRunners.mockResolvedValue(); + mockGetRunnerStateStore.mockReturnValue(undefined); + vi.mocked(mockRunnerStateStore.list).mockResolvedValue([]); + vi.mocked(mockRunnerStateStore.markOrphan).mockResolvedValue(); + vi.mocked(mockRunnerStateStore.unmarkOrphan).mockResolvedValue(); + vi.mocked(mockRunnerStateStore.activate).mockResolvedValue(); + vi.mocked(mockRunnerStateStore.beginTermination).mockResolvedValue('active'); + vi.mocked(mockRunnerStateStore.cancelTermination).mockResolvedValue(); + vi.mocked(mockRunnerStateStore.delete).mockResolvedValue(); mockOctokit.apps.getOrgInstallation.mockImplementation(() => ({ data: { @@ -691,6 +720,309 @@ describe('Scale down runners', () => { }); }); }); + + describe('with durable runner inventory', () => { + beforeEach(() => { + mockGetRunnerStateStore.mockReturnValue(mockRunnerStateStore); + mockOctokit.actions.getSelfHostedRunnerForOrg.mockResolvedValue({ + data: { id: 101, name: 'runner', busy: false, status: 'online' }, + }); + mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockResolvedValue({ status: 204 }); + }); + + it('reconciles stored-only runners without terminating compute and preserves provider-only bypass removal', async () => { + const providerRunner = createRunnerTestData( + 'provider-only', + 'Org', + MINIMUM_TIME_RUNNING_IN_MINUTES + 1, + true, + false, + false, + ); + providerRunner.bypassRemoval = true; + const storedRecord = createRunnerStateRecord('i-stored-only-org', 'active', { + githubRunnerId: '101', + }); + + mockProviderRunners([providerRunner]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValueOnce([storedRecord]).mockResolvedValueOnce([]); + await scaleDown(); + + expect(mockTerminateRunners).not.toHaveBeenCalledWith(storedRecord.computeResourceId); + expect(mockTerminateRunners).not.toHaveBeenCalledWith(providerRunner.id); + expect(mockOctokit.actions.deleteSelfHostedRunnerFromOrg).toHaveBeenCalledWith({ + org: storedRecord.runnerOwner, + runner_id: 101, + }); + expect(mockRunnerStateStore.delete).toHaveBeenCalledWith(storedRecord.runnerId); + }); + + it('keeps inventory lifecycle canonical when a tracked provider resource has a stale orphan tag', async () => { + const providerRunner = createRunnerTestData('tracked-active', 'Org', MINIMUM_BOOT_TIME - 1, false, true, false); + const storedRecord = createRunnerStateRecord(providerRunner.id, 'active'); + + mockProviderRunners([providerRunner]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValue([storedRecord]); + mockOctokit.paginate.mockResolvedValue([]); + + await scaleDown(); + + expect(mockTerminateRunners).not.toHaveBeenCalled(); + expect(mockUnmarkOrphan).not.toHaveBeenCalled(); + expect(mockRunnerStateStore.beginTermination).not.toHaveBeenCalled(); + }); + + it('claims provider-absent active state before GitHub cleanup and deletes it without compute calls', async () => { + const storedRecord = createRunnerStateRecord('i-active-org', 'active', { githubRunnerId: '101' }); + mockProviderRunners([]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValueOnce([storedRecord]).mockResolvedValueOnce([]); + vi.mocked(mockRunnerStateStore.beginTermination).mockResolvedValueOnce('active'); + await scaleDown(); + + expect(mockRunnerStateStore.beginTermination).toHaveBeenCalledWith(storedRecord.runnerId); + expect(mockTerminateRunners).not.toHaveBeenCalled(); + expect(mockRunnerStateStore.delete).toHaveBeenCalledWith(storedRecord.runnerId); + expect(vi.mocked(mockRunnerStateStore.beginTermination).mock.invocationCallOrder[0]).toBeLessThan( + mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mock.invocationCallOrder[0], + ); + expect(mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(mockRunnerStateStore.delete).mock.invocationCallOrder[0], + ); + }); + + it('treats a GitHub 404 as successful cleanup for provider-absent inventory', async () => { + const storedRecord = createRunnerStateRecord('i-missing-github-runner-org', 'provisioning', { + githubRunnerId: '101', + }); + const error404 = new RequestError('Runner not found', 404, { + request: { + method: 'DELETE', + url: 'https://api.github.com/test', + headers: {}, + }, + }); + mockProviderRunners([]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValueOnce([storedRecord]).mockResolvedValueOnce([]); + vi.mocked(mockRunnerStateStore.beginTermination).mockResolvedValueOnce('provisioning'); + mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockRejectedValueOnce(error404); + + await scaleDown(); + + expect(mockRunnerStateStore.delete).toHaveBeenCalledWith(storedRecord.runnerId); + expect(mockRunnerStateStore.cancelTermination).not.toHaveBeenCalled(); + expect(mockTerminateRunners).not.toHaveBeenCalled(); + expect(mockUnmarkOrphan).not.toHaveBeenCalled(); + expect(mockOctokit.actions.getSelfHostedRunnerForOrg).not.toHaveBeenCalled(); + }); + + it('skips cleanup when another invocation owns the termination claim', async () => { + const storedRecord = createRunnerStateRecord('i-contended-org', 'active', { githubRunnerId: '101' }); + mockProviderRunners([]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValue([storedRecord]); + vi.mocked(mockRunnerStateStore.beginTermination).mockResolvedValueOnce(undefined); + mockOctokit.paginate.mockResolvedValue([{ id: 101, name: storedRecord.computeResourceId }]); + + await scaleDown(); + + expect(mockOctokit.actions.deleteSelfHostedRunnerFromOrg).not.toHaveBeenCalled(); + expect(mockTerminateRunners).not.toHaveBeenCalled(); + expect(mockRunnerStateStore.delete).not.toHaveBeenCalled(); + }); + + it('reclaims a stale terminating provider-absent record without calling compute', async () => { + const terminatingRecord = createRunnerStateRecord('i-stale-terminating-org', 'terminating'); + mockProviderRunners([]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValueOnce([terminatingRecord]).mockResolvedValueOnce([]); + vi.mocked(mockRunnerStateStore.beginTermination).mockResolvedValueOnce('terminating'); + + await scaleDown(); + + expect(mockRunnerStateStore.beginTermination).toHaveBeenCalledWith(terminatingRecord.runnerId); + expect(mockTerminateRunners).not.toHaveBeenCalled(); + expect(mockRunnerStateStore.delete).toHaveBeenCalledWith(terminatingRecord.runnerId); + expect(mockRunnerStateStore.cancelTermination).not.toHaveBeenCalled(); + }); + + it('leaves a reclaimed provider-absent record terminating when state deletion fails', async () => { + const terminatingRecord = createRunnerStateRecord('i-terminating-retry-failure-org', 'terminating'); + mockProviderRunners([]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValue([terminatingRecord]); + vi.mocked(mockRunnerStateStore.beginTermination).mockResolvedValueOnce('terminating'); + vi.mocked(mockRunnerStateStore.delete).mockRejectedValueOnce(new Error('state deletion failed')); + + await scaleDown(); + + expect(mockRunnerStateStore.cancelTermination).not.toHaveBeenCalled(); + expect(mockRunnerStateStore.delete).toHaveBeenCalledWith(terminatingRecord.runnerId); + expect(mockTerminateRunners).not.toHaveBeenCalled(); + }); + + it('restores the exact prior lifecycle state when compute termination fails', async () => { + const providerRunner = createRunnerTestData( + 'stale-provisioning', + 'Org', + MINIMUM_BOOT_TIME + 1, + false, + false, + false, + ); + const staleProvisioning = createRunnerStateRecord(providerRunner.id, 'provisioning'); + mockProviderRunners([providerRunner]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValue([staleProvisioning]); + vi.mocked(mockRunnerStateStore.beginTermination).mockResolvedValueOnce('provisioning'); + mockTerminateRunners.mockRejectedValueOnce(new Error('termination failed')); + + await scaleDown(); + + expect(mockRunnerStateStore.beginTermination).toHaveBeenCalledWith(staleProvisioning.runnerId); + expect(mockRunnerStateStore.cancelTermination).toHaveBeenCalledWith(staleProvisioning.runnerId, 'provisioning'); + expect(mockRunnerStateStore.delete).not.toHaveBeenCalled(); + }); + + it('de-registers a stored orphan before terminating compute and deleting state', async () => { + const providerRunner = createRunnerTestData( + 'orphan-with-github-runner', + 'Org', + MINIMUM_BOOT_TIME + 1, + false, + true, + true, + ); + const storedOrphan = createRunnerStateRecord(providerRunner.id, 'orphan', { + githubRunnerId: '101', + }); + mockProviderRunners([providerRunner]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValueOnce([storedOrphan]).mockResolvedValueOnce([]); + vi.mocked(mockRunnerStateStore.beginTermination).mockResolvedValueOnce('orphan'); + mockOctokit.actions.getSelfHostedRunnerForOrg.mockResolvedValueOnce({ + data: { id: 101, name: storedOrphan.computeResourceId, busy: true, status: 'offline' }, + }); + + await scaleDown(); + + expect(mockOctokit.actions.deleteSelfHostedRunnerFromOrg).toHaveBeenCalledWith({ + org: storedOrphan.runnerOwner, + runner_id: 101, + }); + expect(mockTerminateRunners).toHaveBeenCalledWith(storedOrphan.computeResourceId); + expect(mockRunnerStateStore.delete).toHaveBeenCalledWith(storedOrphan.runnerId); + expect(mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mock.invocationCallOrder[0]).toBeLessThan( + mockTerminateRunners.mock.invocationCallOrder[0], + ); + expect(mockTerminateRunners.mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(mockRunnerStateStore.delete).mock.invocationCallOrder[0], + ); + }); + + it('restores an orphan claim when GitHub cleanup fails', async () => { + const storedOrphan = createRunnerStateRecord('i-orphan-cleanup-failure-org', 'orphan', { + githubRunnerId: '101', + }); + mockProviderRunners([]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValue([storedOrphan]); + vi.mocked(mockRunnerStateStore.beginTermination).mockResolvedValueOnce('orphan'); + mockOctokit.actions.deleteSelfHostedRunnerFromOrg.mockResolvedValueOnce({ status: 500 }); + + await scaleDown(); + + expect(mockRunnerStateStore.cancelTermination).toHaveBeenCalledWith(storedOrphan.runnerId, 'orphan'); + expect(mockTerminateRunners).not.toHaveBeenCalled(); + expect(mockRunnerStateStore.delete).not.toHaveBeenCalled(); + }); + + it('does not restore state when deletion fails after compute termination succeeds', async () => { + const providerRunner = createRunnerTestData( + 'delete-failure', + 'Org', + MINIMUM_TIME_RUNNING_IN_MINUTES + 1, + true, + false, + true, + ); + const storedRecord = createRunnerStateRecord(providerRunner.id, 'active', { githubRunnerId: '101' }); + mockProviderRunners([providerRunner]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValue([storedRecord]); + vi.mocked(mockRunnerStateStore.beginTermination).mockResolvedValueOnce('active'); + vi.mocked(mockRunnerStateStore.delete).mockRejectedValueOnce(new Error('delete failed')); + mockOctokit.paginate.mockResolvedValue([{ id: 101, name: storedRecord.computeResourceId }]); + + await expect(scaleDown()).resolves.not.toThrow(); + + expect(mockTerminateRunners).toHaveBeenCalledWith(providerRunner.id); + expect(mockRunnerStateStore.cancelTermination).not.toHaveBeenCalled(); + }); + + it('updates durable lifecycle only after provider orphan removal succeeds', async () => { + const providerRunner = createRunnerTestData( + 'tracked-orphan', + 'Org', + MINIMUM_BOOT_TIME + 1, + false, + true, + false, + undefined, + 101, + ); + const storedOrphan = createRunnerStateRecord(providerRunner.id, 'orphan', { githubRunnerId: '101' }); + mockProviderRunners([providerRunner]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValue([storedOrphan]); + mockOctokit.actions.getSelfHostedRunnerForOrg.mockResolvedValueOnce({ + data: { id: 101, name: providerRunner.id, busy: false, status: 'online' }, + }); + + await scaleDown(); + + expect(mockUnmarkOrphan).toHaveBeenCalledWith(providerRunner.id); + expect(mockRunnerStateStore.unmarkOrphan).toHaveBeenCalledWith(storedOrphan.runnerId); + expect(mockUnmarkOrphan.mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(mockRunnerStateStore.unmarkOrphan).mock.invocationCallOrder[0], + ); + expect(mockTerminateRunners).not.toHaveBeenCalled(); + }); + + it('retains durable orphan state when provider orphan removal fails', async () => { + const providerRunner = createRunnerTestData( + 'tracked-orphan-failure', + 'Org', + MINIMUM_BOOT_TIME + 1, + false, + true, + false, + undefined, + 101, + ); + const storedOrphan = createRunnerStateRecord(providerRunner.id, 'orphan', { githubRunnerId: '101' }); + mockProviderRunners([providerRunner]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValue([storedOrphan]); + mockOctokit.actions.getSelfHostedRunnerForOrg.mockResolvedValueOnce({ + data: { id: 101, name: providerRunner.id, busy: false, status: 'online' }, + }); + mockUnmarkOrphan.mockRejectedValueOnce(new Error('provider unmark failed')); + + await scaleDown(); + + expect(mockRunnerStateStore.unmarkOrphan).not.toHaveBeenCalled(); + expect(mockRunnerStateStore.activate).not.toHaveBeenCalled(); + expect(mockTerminateRunners).not.toHaveBeenCalled(); + }); + + it('reconciles expired provisioning records but leaves fresh provisioning records alone', async () => { + const staleProvisioning = createRunnerStateRecord('i-stale-provisioning-org', 'provisioning'); + const freshProvisioning = createRunnerStateRecord('i-fresh-provisioning-org', 'provisioning', { + createdAt: new Date().toISOString(), + }); + mockProviderRunners([]); + vi.mocked(mockRunnerStateStore.list).mockResolvedValue([staleProvisioning, freshProvisioning]); + vi.mocked(mockRunnerStateStore.beginTermination).mockResolvedValueOnce('provisioning'); + + await scaleDown(); + + expect(mockRunnerStateStore.beginTermination).toHaveBeenCalledTimes(1); + expect(mockRunnerStateStore.beginTermination).toHaveBeenCalledWith(staleProvisioning.runnerId); + expect(mockTerminateRunners).not.toHaveBeenCalled(); + expect(mockRunnerStateStore.delete).toHaveBeenCalledWith(staleProvisioning.runnerId); + }); + }); }); function mockProviderRunners(runners: RunnerTestItem[]) { @@ -751,3 +1083,24 @@ function createRunnerTestData( bypassRemoval: false, }; } + +function createRunnerStateRecord( + runnerId: string, + state: RunnerStateRecord['state'], + overrides: Partial = {}, +): RunnerStateRecord { + const timestamp = moment(new Date()) + .subtract(MINIMUM_TIME_RUNNING_IN_MINUTES + 1, 'minutes') + .toISOString(); + return { + runnerId, + computeProvider: 'ec2', + computeResourceId: runnerId, + runnerOwner: TEST_DATA.repositoryOwner, + runnerType: 'Org', + state, + createdAt: timestamp, + updatedAt: timestamp, + ...overrides, + }; +} diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts index 329ce694d9..3c49158a44 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-down.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-down.ts @@ -3,6 +3,11 @@ import { Endpoints } from '@octokit/types'; import { RequestError } from '@octokit/request-error'; import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; import { resolveComputeProviderType } from '@aws-github-runner/compute-providers/provider-types'; +import { + getRunnerStateStore, + type RunnerStateRecord, + type RunnerStateStore, +} from '@aws-github-runner/storage-providers'; import moment from 'moment'; import { @@ -22,7 +27,19 @@ const logger = createChildLogger('scale-down'); type OrgRunnerList = Endpoints['GET /orgs/{org}/actions/runners']['response']['data']['runners']; type RepoRunnerList = Endpoints['GET /repos/{owner}/{repo}/actions/runners']['response']['data']['runners']; -type RunnerState = OrgRunnerList[number] | RepoRunnerList[number]; +type GitHubRunnerState = OrgRunnerList[number] | RepoRunnerList[number]; + +interface InventoryRunnerInfo extends RunnerInfo { + inventory?: RunnerStateRecord; + providerPresent?: boolean; +} + +type RestorableRunnerState = Parameters[1]; + +interface TerminationClaim { + runnerStateId?: string; + restoreState?: RestorableRunnerState; +} async function getOrCreateOctokit(runner: RunnerInfo): Promise { const key = runner.owner; @@ -67,7 +84,7 @@ async function getGitHubSelfHostedRunnerState( client: Octokit, runner: RunnerInfo, runnerId: number, -): Promise { +): Promise { try { const state = runner.type === 'Org' @@ -161,6 +178,12 @@ async function deleteGitHubRunner( } return { ghRunnerId, status: response.status, success: response.status === 204 }; } catch (error) { + if (error instanceof RequestError && error.status === 404) { + logger.info( + `GitHub runner ${ghRunnerId} for runner '${runner.id}' is already de-registered; treating cleanup as complete.`, + ); + return { ghRunnerId, status: error.status, success: true }; + } logger.error( `Failed to de-register GitHub runner ${ghRunnerId} for runner '${runner.id}'. ` + `Error: ${error instanceof Error ? error.message : String(error)}`, @@ -171,11 +194,12 @@ async function deleteGitHubRunner( } async function removeRunner( - runner: RunnerInfo, + runner: InventoryRunnerInfo, ghRunnerIds: number[], computeProvider: ScaleDownComputeProvider, ): Promise { - const githubInstallationClient = await getOrCreateOctokit(runner); + let terminationClaim: TerminationClaim | undefined; + let computeTerminated = false; try { if (runner.bypassRemoval) { logger.info( @@ -184,6 +208,7 @@ async function removeRunner( return; } + const githubInstallationClient = await getOrCreateOctokit(runner); const states = await Promise.all( ghRunnerIds.map(async (ghRunnerId) => { // Get busy state instead of using the output of listGitHubRunners(...) to minimize to race condition. @@ -192,6 +217,13 @@ async function removeRunner( ); if (states.every((busy) => busy === false)) { + const claim = await beginRunnerTermination(runner); + if (claim === undefined) { + logger.info(`Runner '${runner.id}' is already being reconciled; skipping this scale-down cycle.`); + return; + } + terminationClaim = claim; + const results = await Promise.all( ghRunnerIds.map((ghRunnerId) => deleteGitHubRunner(githubInstallationClient, runner, ghRunnerId)), ); @@ -201,10 +233,15 @@ async function removeRunner( if (allSucceeded) { await computeProvider.terminate(runner.id); + computeTerminated = true; + await completeRunnerTermination(terminationClaim); + terminationClaim = undefined; logger.info( `${computeProvider.type.toUpperCase()} runner '${runner.id}' is terminated and GitHub runner is de-registered.`, ); } else { + await restoreRunnerTermination(terminationClaim); + terminationClaim = undefined; // Only terminate the provider runner if it was successfully de-registered from GitHub. logger.error( `Failed to de-register ${failedRunners.length} GitHub runner(s) for runner '${runner.id}'. ` + @@ -216,6 +253,9 @@ async function removeRunner( logger.info(`Runner '${runner.id}' cannot be de-registered, because it is still busy.`); } } catch (e) { + if (terminationClaim && !computeTerminated) { + await restoreRunnerTermination(terminationClaim); + } logger.error( `Runner '${runner.id}' cannot be de-registered. Error: ${e instanceof Error ? e.message : String(e)}`, { error: e }, @@ -224,7 +264,7 @@ async function removeRunner( } async function evaluateAndRemoveRunners( - runners: RunnerInfo[], + runners: InventoryRunnerInfo[], scaleDownConfigs: ScalingDownConfigList, computeProvider: ScaleDownComputeProvider, ): Promise { @@ -237,14 +277,27 @@ async function evaluateAndRemoveRunners( .filter((runner) => runner.owner === ownerTag) .sort(evictionStrategy === 'oldest_first' ? oldestFirstStrategy : newestFirstStrategy); logger.debug(`Found: '${ownerRunners.length}' active GitHub runners with owner tag: '${ownerTag}'`); - logger.debug(`Active GitHub runners with owner tag: '${ownerTag}': ${JSON.stringify(ownerRunners)}`); + if (ownerRunners.some((runner) => runner.inventory)) { + logger.debug(`Active GitHub runner inventory with owner tag: '${ownerTag}'`, { + runners: ownerRunners.map((runner) => ({ + computeResourceId: runner.id, + lifecycleState: runner.inventory?.state, + })), + }); + } else { + logger.debug(`Active GitHub runners with owner tag: '${ownerTag}': ${JSON.stringify(ownerRunners)}`); + } for (const runner of ownerRunners) { if (runner.bypassRemoval) { logger.debug(`Runner '${runner.id}' has bypass-removal tag set, skipping evaluation.`); continue; } const ghRunners = await listGitHubRunners(runner); - const ghRunnersFiltered = ghRunners.filter((ghRunner: { name: string }) => ghRunner.name.endsWith(runner.id)); + const ghRunnersFiltered = ghRunners.filter((ghRunner: { id: number; name: string }) => + runner.inventory?.githubRunnerId + ? ghRunner.id.toString() === runner.inventory.githubRunnerId + : ghRunner.name.endsWith(runner.id), + ); logger.debug(`Found: '${ghRunnersFiltered.length}' GitHub runners for runner: '${runner.id}'`); logger.debug(`GitHub runners for runner: '${runner.id}': ${JSON.stringify(ghRunnersFiltered)}`); if (ghRunnersFiltered.length) { @@ -262,7 +315,7 @@ async function evaluateAndRemoveRunners( } } } else if (computeProvider.bootTimeExceeded(runner)) { - await markOrphan(runner.id, computeProvider); + await markOrphan(runner, computeProvider); } else { logger.debug(`Runner ${runner.id} has not yet booted.`); } @@ -270,25 +323,46 @@ async function evaluateAndRemoveRunners( } } -async function markOrphan(id: string, computeProvider: ScaleDownComputeProvider): Promise { +async function markOrphan(runner: InventoryRunnerInfo, computeProvider: ScaleDownComputeProvider): Promise { try { - await computeProvider.markOrphan(id); - logger.info(`Runner '${id}' tagged as orphan.`); + if (runner.inventory) { + await getRunnerStateStore()?.markOrphan(runner.inventory.runnerId); + } + await computeProvider.markOrphan(runner.id); + logger.info(`Runner '${runner.id}' tagged as orphan.`); } catch (e) { - logger.error(`Failed to tag runner '${id}' as orphan.`, { error: e }); + logger.error(`Failed to tag runner '${runner.id}' as orphan.`, { error: e }); } } -async function unMarkOrphan(id: string, computeProvider: ScaleDownComputeProvider): Promise { +async function unMarkOrphan(runner: InventoryRunnerInfo, computeProvider: ScaleDownComputeProvider): Promise { try { - await computeProvider.unmarkOrphan(id); - logger.info(`Runner '${id}' untagged as orphan.`); + // Keep the durable lifecycle unchanged if the provider mutation fails. In + // particular, activating a provisioning record removes its safety TTL. + await computeProvider.unmarkOrphan(runner.id); + const runnerStateStore = getRunnerStateStore(); + if (runnerStateStore && runner.inventory?.state === 'orphan') { + await runnerStateStore.unmarkOrphan(runner.inventory.runnerId); + } else if (runnerStateStore && runner.inventory?.state === 'provisioning') { + await runnerStateStore.activate(runner.inventory.runnerId, { + githubRunnerId: runner.githubRunnerId, + runnerLabels: runner.inventory.runnerLabels, + runnerName: runner.inventory.runnerName, + metadata: runner.inventory.metadata, + }); + } + logger.info(`Runner '${runner.id}' untagged as orphan.`); } catch (e) { - logger.error(`Failed to un-tag runner '${id}' as orphan.`, { error: e }); + logger.error(`Failed to un-tag runner '${runner.id}' as orphan.`, { error: e }); } } -async function lastChanceCheckOrphanRunner(runner: RunnerInfo): Promise { +interface OrphanEvaluation { + isOrphan: boolean; + githubRunnerExists: boolean; +} + +async function lastChanceCheckOrphanRunner(runner: InventoryRunnerInfo): Promise { const client = await getOrCreateOctokit(runner); const runnerId = parseInt(runner.githubRunnerId || '0'); const state = await getGitHubSelfHostedRunnerState(client, runner, runnerId); @@ -305,30 +379,52 @@ async function lastChanceCheckOrphanRunner(runner: RunnerInfo): Promise } } logger.info(`Runner '${runner.id}' is judged to ${isOrphan ? 'be' : 'not be'} orphaned.`); - return isOrphan; + return { isOrphan, githubRunnerExists: state !== null }; } -async function terminateOrphan(environment: string, computeProvider: ScaleDownComputeProvider): Promise { +async function terminateOrphan( + environment: string, + computeProvider: ScaleDownComputeProvider, + runners?: InventoryRunnerInfo[], +): Promise { try { - const orphanRunners = await computeProvider.list(environment, true); + const orphanRunners: InventoryRunnerInfo[] = runners ?? (await computeProvider.list(environment, true)); for (const runner of orphanRunners) { if (runner.bypassRemoval) { logger.info(`Orphan runner '${runner.id}' has bypass-removal tag set, skipping termination.`); continue; } + if (runner.inventory?.state === 'provisioning' && !computeProvider.bootTimeExceeded(runner)) { + logger.debug(`Runner '${runner.id}' is still provisioning; skipping reconciliation until boot time expires.`); + continue; + } if (runner.githubRunnerId) { - const isOrphan = await lastChanceCheckOrphanRunner(runner); - if (isOrphan) { - await computeProvider.terminate(runner.id); + const orphanEvaluation = await lastChanceCheckOrphanRunner(runner); + if (orphanEvaluation.isOrphan) { + if (runner.inventory) { + await terminateClaimedRunner( + runner, + computeProvider, + orphanEvaluation.githubRunnerExists ? parseInt(runner.githubRunnerId) : undefined, + ); + } else { + // Preserve the provider-only recovery behavior used by the legacy path. + await computeProvider.terminate(runner.id); + } } else { - await unMarkOrphan(runner.id, computeProvider); + await unMarkOrphan(runner, computeProvider); } } else { logger.info(`Terminating orphan runner '${runner.id}'`); - await computeProvider.terminate(runner.id).catch((e) => { - logger.error(`Failed to terminate orphan runner '${runner.id}'`, { error: e }); - }); + if (runner.inventory) { + await terminateClaimedRunner(runner, computeProvider); + } else { + // Preserve the provider-only recovery behavior used by the legacy path. + await computeProvider.terminate(runner.id).catch((error) => { + logger.error(`Failed to terminate orphan runner '${runner.id}'`, { error }); + }); + } } } } catch (e) { @@ -336,6 +432,136 @@ async function terminateOrphan(environment: string, computeProvider: ScaleDownCo } } +async function terminateClaimedRunner( + runner: InventoryRunnerInfo, + computeProvider: ScaleDownComputeProvider, + githubRunnerId?: number, +): Promise { + const claim = await beginRunnerTermination(runner); + if (claim === undefined) { + logger.info(`Runner '${runner.id}' is already being reconciled; skipping this scale-down cycle.`); + return; + } + + try { + if (githubRunnerId !== undefined) { + const githubInstallationClient = await getOrCreateOctokit(runner); + const result = await deleteGitHubRunner(githubInstallationClient, runner, githubRunnerId); + if (!result.success) { + await restoreRunnerTermination(claim); + logger.error( + `Failed to de-register GitHub runner '${githubRunnerId}' for orphan runner '${runner.id}'. ` + + `Runner will NOT be terminated to allow retry on next scale-down cycle.`, + ); + return; + } + } + await computeProvider.terminate(runner.id); + } catch (error) { + await restoreRunnerTermination(claim); + logger.error(`Failed to clean up orphan runner '${runner.id}'`, { error }); + return; + } + + try { + await completeRunnerTermination(claim); + } catch (error) { + logger.error(`Failed to delete runner state for terminated runner '${runner.id}'`, { error }); + } +} + +async function reconcileProviderAbsentRunner( + runner: InventoryRunnerInfo, + computeProvider: ScaleDownComputeProvider, +): Promise { + if (!runner.inventory || runner.providerPresent !== false) { + return; + } + + // Provider discovery can lag immediately after launch. Retain the inventory + // until the normal boot grace has elapsed before declaring compute absent. + if (!computeProvider.bootTimeExceeded(runner)) { + logger.debug(`Runner '${runner.id}' is absent from provider discovery but remains inside its boot grace.`); + return; + } + + const claim = await beginRunnerTermination(runner); + if (claim === undefined) { + logger.info(`Runner '${runner.id}' is already being reconciled; skipping this scale-down cycle.`); + return; + } + + try { + if (runner.githubRunnerId) { + const githubInstallationClient = await getOrCreateOctokit(runner); + const result = await deleteGitHubRunner(githubInstallationClient, runner, parseInt(runner.githubRunnerId, 10)); + if (!result.success) { + await restoreRunnerTermination(claim); + logger.error( + `Failed to de-register GitHub runner '${runner.githubRunnerId}' for provider-absent runner ` + + `'${runner.id}'. Runner state will be retained for retry.`, + ); + return; + } + } + } catch (error) { + await restoreRunnerTermination(claim); + logger.error(`Failed to reconcile GitHub state for provider-absent runner '${runner.id}'`, { error }); + return; + } + + // Provider discovery already established that the compute resource is gone; + // avoid tag/terminate calls that would turn idempotent cleanup into a retry loop. + try { + await completeRunnerTermination(claim); + logger.info(`Removed durable state for provider-absent runner '${runner.id}'.`); + } catch (error) { + // GitHub cleanup is committed at this point. Leave the terminating claim in + // place so its lease/TTL makes state deletion retryable without resurrection. + logger.error(`Failed to delete runner state for provider-absent runner '${runner.id}'`, { error }); + } +} + +async function beginRunnerTermination(runner: InventoryRunnerInfo): Promise { + if (!runner.inventory) { + return {}; + } + + const runnerStateStore = getRunnerStateStore(); + if (!runnerStateStore) { + return {}; + } + + const previousState = await runnerStateStore.beginTermination(runner.inventory.runnerId); + if (previousState === undefined) { + return undefined; + } + return { + runnerStateId: runner.inventory.runnerId, + // A reclaimed terminating record has no known pre-claim lifecycle state. + // Leave it terminating on failure so the lease can make it retryable again. + ...(previousState === 'terminating' ? {} : { restoreState: previousState }), + }; +} + +async function restoreRunnerTermination(claim: TerminationClaim): Promise { + if (!claim.runnerStateId || !claim.restoreState) { + return; + } + + try { + await getRunnerStateStore()?.cancelTermination(claim.runnerStateId, claim.restoreState); + } catch (error) { + logger.error(`Failed to restore runner state for '${claim.runnerStateId}' after termination failure.`, { error }); + } +} + +async function completeRunnerTermination(claim: TerminationClaim): Promise { + if (claim.runnerStateId) { + await getRunnerStateStore()?.delete(claim.runnerStateId); + } +} + export function oldestFirstStrategy(a: RunnerInfo, b: RunnerInfo): number { if (a.launchTime === undefined) return 1; if (b.launchTime === undefined) return 1; @@ -352,12 +578,52 @@ async function listRunners(environment: string, computeProvider: ScaleDownComput return await computeProvider.list(environment); } -function filterRunners(runners: RunnerInfo[]): RunnerInfo[] { +function filterRunners(runners: InventoryRunnerInfo[]): InventoryRunnerInfo[] { // Managed runners are launched with owner and type tags together. Exclude incomplete records because both // values are required to select the GitHub owner and runner API used during scale-down. return runners.filter((runner) => runner.owner && runner.type && !runner.orphan); } +function mergeRunnerInventory(records: RunnerStateRecord[], providerRunners: RunnerInfo[]): InventoryRunnerInfo[] { + const runnersByComputeResource = new Map(); + for (const record of records) { + runnersByComputeResource.set(record.computeResourceId, { + id: record.computeResourceId, + launchTime: new Date(record.createdAt), + owner: record.runnerOwner, + type: record.runnerType, + orphan: record.state !== 'active', + githubRunnerId: record.githubRunnerId, + inventory: record, + providerPresent: false, + }); + } + + for (const providerRunner of providerRunners) { + const storedRunner = runnersByComputeResource.get(providerRunner.id); + if (!storedRunner) { + runnersByComputeResource.set(providerRunner.id, { ...providerRunner, providerPresent: true }); + continue; + } + + runnersByComputeResource.set(providerRunner.id, { + ...storedRunner, + ...providerRunner, + id: storedRunner.id, + owner: storedRunner.owner, + type: storedRunner.type, + // Lifecycle state is canonical for tracked resources. Provider tags are + // retained as recovery data only for resources missing from inventory. + orphan: storedRunner.orphan, + githubRunnerId: storedRunner.githubRunnerId ?? providerRunner.githubRunnerId, + inventory: storedRunner.inventory, + providerPresent: true, + }); + } + + return Array.from(runnersByComputeResource.values()); +} + export async function scaleDown(): Promise { githubCache.reset(); const environment = process.env.ENVIRONMENT; @@ -367,28 +633,70 @@ export async function scaleDown(): Promise { ...controlPlaneProviderRegistry.capability(computeProviderType, 'scaleDown')(), type: computeProviderType, }; + const runnerStateStore = getRunnerStateStore(); + let managedRunners: InventoryRunnerInfo[]; + let runnersToEvaluate: InventoryRunnerInfo[]; + + if (!runnerStateStore) { + // Preserve the legacy SSM-backed lifecycle. EC2 remains the source of truth + // when no durable runner inventory is configured. + await terminateOrphan(environment, computeProvider); + managedRunners = await listRunners(environment, computeProvider); + runnersToEvaluate = managedRunners; + } else { + const providerRunners = await listRunners(environment, computeProvider); + const inventoryRecords = await runnerStateStore.list({ computeProvider: computeProvider.type }); + managedRunners = mergeRunnerInventory(inventoryRecords, providerRunners); + + const providerAbsentRunners = managedRunners.filter( + (runner) => runner.inventory !== undefined && runner.providerPresent === false, + ); + for (const runner of providerAbsentRunners) { + await reconcileProviderAbsentRunner(runner, computeProvider); + } + runnersToEvaluate = managedRunners.filter( + (runner) => runner.inventory === undefined || runner.providerPresent !== false, + ); - // first runners marked to be orphan. - await terminateOrphan(environment, computeProvider); + // Reconcile stale provisioning and orphan records first. Provider tags remain + // a recovery signal for resources that predate the durable inventory. + await terminateOrphan( + environment, + computeProvider, + runnersToEvaluate.filter((runner) => runner.orphan), + ); + } - // next scale down idle runners with respect to config and mark potential orphans - const providerRunners = await listRunners(environment, computeProvider); - const activeProviderRunnersCount = providerRunners.length; + const runnerCountLabel = runnerStateStore ? 'managed' : 'active'; + const managedRunnerCount = managedRunners.length; logger.info( - `Found: '${activeProviderRunnersCount}' active ${computeProvider.type.toUpperCase()} runners before clean-up.`, + `Found: '${managedRunnerCount}' ${runnerCountLabel} ${computeProvider.type.toUpperCase()} runners before clean-up.`, ); - logger.debug(`Active ${computeProvider.type.toUpperCase()} runners: ${JSON.stringify(providerRunners)}`); + if (runnerStateStore) { + logger.debug(`Active ${computeProvider.type.toUpperCase()} runner inventory`, { + runners: managedRunners.map((runner) => ({ + computeResourceId: runner.id, + lifecycleState: runner.inventory?.state, + })), + }); + } else { + logger.debug(`Active ${computeProvider.type.toUpperCase()} runners: ${JSON.stringify(managedRunners)}`); + } - if (activeProviderRunnersCount === 0) { - logger.debug(`No active runners found for environment: '${environment}'`); + if (managedRunnerCount === 0) { + logger.debug(`No ${runnerCountLabel} runners found for environment: '${environment}'`); return; } - const runners = filterRunners(providerRunners); + const runners = filterRunners(runnersToEvaluate); await evaluateAndRemoveRunners(runners, scaleDownConfigs, computeProvider); - const activeProviderRunnersCountAfter = (await listRunners(environment, computeProvider)).length; + const providerRunnersAfter = await listRunners(environment, computeProvider); + const managedRunnerCountAfter = runnerStateStore + ? mergeRunnerInventory(await runnerStateStore.list({ computeProvider: computeProvider.type }), providerRunnersAfter) + .length + : providerRunnersAfter.length; logger.info( - `Found: '${activeProviderRunnersCountAfter}' active ${computeProvider.type.toUpperCase()} runners after clean-up.`, + `Found: '${managedRunnerCountAfter}' ${runnerCountLabel} ${computeProvider.type.toUpperCase()} runners after clean-up.`, ); } diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up-contract.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up-contract.test.ts index d7a025fd88..055ba0db5c 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up-contract.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up-contract.test.ts @@ -15,6 +15,15 @@ vi.mock('../github/auth', () => ({ createOctokitClient: vi.fn(), })); +vi.mock('@aws-github-runner/storage-providers', () => ({ + createStorageProviders: vi.fn().mockReturnValue({ + githubAppCredentials: { get: vi.fn() }, + }), + getRunnerConfigStore: vi.fn(), + getRunnerGroupCacheStore: vi.fn(), + getRunnerStateStore: vi.fn().mockReturnValue(undefined), +})); + vi.mock('./github-runner', async (importOriginal) => ({ ...(await importOriginal()), getGitHubEnterpriseApiUrl: vi.fn(), diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts index 664bac60fb..f3c93a4227 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts @@ -52,7 +52,7 @@ interface TestRunnerLookupInput { } const createRunner = vi.fn<(input: TestRunnerCreationInput) => Promise>(); -const listRunners = vi.fn<(input: TestRunnerLookupInput) => Promise>(); +const listRunners = vi.fn<(input: TestRunnerLookupInput) => Promise<{ id: string }[]>>(); const mockCreateRunner = vi.mocked(createRunner); const mockListRunners = vi.mocked(listRunners); const mockSSMClient = mockClient(SSMClient); @@ -207,7 +207,7 @@ beforeEach(() => { runnerType: input.runnerType, runnerOwner: input.runnerOwner, }) - ).length; + ).map(({ id }) => id); }); mockCreateRunners.mockImplementation(createTestProviderRunners); diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts index d4e3889f19..719bed0de6 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts @@ -1,7 +1,11 @@ import { addPersistentContextToChildLogger, createChildLogger } from '@aws-github-runner/aws-powertools-util'; import { InvalidRunnerLabelsError } from '@aws-github-runner/compute-providers/core'; import { resolveComputeProviderType } from '@aws-github-runner/compute-providers/provider-types'; -import { createStorageProviders, type StorageProviders } from '@aws-github-runner/storage-providers'; +import { + createStorageProviders, + getRunnerStateStore, + type StorageProviders, +} from '@aws-github-runner/storage-providers'; import { Octokit } from '@octokit/rest'; import yn from 'yn'; @@ -289,7 +293,13 @@ export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise + await computeProvider.getCurrentRunners(runnerLabelResolution.state, { runnerType, runnerOwner }), + ); logger.info('Current runners', { currentRunners, @@ -400,6 +410,31 @@ export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise Promise, +): Promise { + const runnerStateStore = getRunnerStateStore(); + if (!runnerStateStore) { + return (await getProviderRunnerIds()).length; + } + + const records = await runnerStateStore.list({ computeProvider }); + const runnerIds = new Set( + records + .filter((record) => record.runnerType === runnerType && record.runnerOwner === runnerOwner) + .map((record) => record.computeResourceId), + ); + // Durable inventory is canonical. Provider discovery remains a conservative + // recovery source for resources launched before their inventory record was written. + for (const runnerId of await getProviderRunnerIds()) { + runnerIds.add(runnerId); + } + return runnerIds.size; +} + function isValidRepoOwnerTypeIfOrgLevelEnabled(payload: ActionRequestMessage, enableOrgLevel: boolean): boolean { return !(enableOrgLevel && payload.repoOwnerType !== 'Organization'); } diff --git a/lambdas/functions/control-plane/src/test/compute-provider-contracts/scale-up.ts b/lambdas/functions/control-plane/src/test/compute-provider-contracts/scale-up.ts index ecab384118..542f01d726 100644 --- a/lambdas/functions/control-plane/src/test/compute-provider-contracts/scale-up.ts +++ b/lambdas/functions/control-plane/src/test/compute-provider-contracts/scale-up.ts @@ -43,7 +43,7 @@ export function defineScaleUpContractTests({ resolveCapability.mockReturnValue(() => provider); vi.mocked(provider.resolveLabelsForRunners).mockResolvedValue({ runnerLabels: [], state }); - vi.mocked(provider.getCurrentRunners).mockResolvedValue(0); + vi.mocked(provider.getCurrentRunners).mockResolvedValue([]); vi.mocked(provider.createRunners).mockResolvedValue(createResult); }); @@ -81,7 +81,7 @@ export function defineScaleUpContractTests({ it('does not create runners when the compute provider has reached maximum capacity', async () => { process.env.RUNNERS_MAXIMUM_COUNT = '1'; - vi.mocked(provider.getCurrentRunners).mockResolvedValue(1); + vi.mocked(provider.getCurrentRunners).mockResolvedValue(['runner-1']); await scaleUp(createPayloads()); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts index 124b676ae6..7b1e41cbbc 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts @@ -155,6 +155,9 @@ function createEc2StartRunnerConfigOptions( return { runnerConfigStore: storage?.runnerConfig, runnerGroupCacheStore: storage?.runnerGroupCache, + computeProvider: 'ec2', + getRunnerConfigAccessScope: (instanceId) => + process.env.EC2_INSTANCE_ARN_PREFIX ? `${process.env.EC2_INSTANCE_ARN_PREFIX}${instanceId}` : undefined, getRunnerConfigMetadata: (instanceId) => [{ key: 'InstanceId', value: instanceId }], onJitConfigCreated: async (instanceId, metadata) => await tagEc2RunnerMetadata(ec2Operations, instanceId, metadata), }; diff --git a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts index 2a79eea40d..663e2ed5b3 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts @@ -106,7 +106,7 @@ async function expectCurrentRunners(runnerType: RunnerType, owner: string) { runnerType, runnerOwner: owner, }), - ).resolves.toBe(1); + ).resolves.toEqual(['i-1234']); expect(mockListRunners).toHaveBeenCalledWith({ environment: 'unit-test-environment', runnerType, diff --git a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.ts index a27758b574..ebc2a6a263 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.ts @@ -59,7 +59,9 @@ export function createEc2ScaleUpCapability( return { resolveLabelsForRunners: (labels) => resolveEc2ScaleUpRunnerLabels(ec2Operations, labels), getCurrentRunners: async (_state, { runnerType, runnerOwner }) => - (await ec2Operations.list({ environment: process.env.ENVIRONMENT, runnerType, runnerOwner })).length, + (await ec2Operations.list({ environment: process.env.ENVIRONMENT, runnerType, runnerOwner })).map( + (runner) => runner.id, + ), createRunners: async ({ githubRunnerConfig, numberOfRunners, githubInstallationClient, state, storage }) => { const config = loadEc2ScaleUpProviderConfig(); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/environment.d.ts b/lambdas/libs/compute-providers/aws/ec2/src/environment.d.ts index 71ee01ff2f..189b161925 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/environment.d.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/environment.d.ts @@ -16,6 +16,7 @@ declare global { | 'capacity-optimized' | 'capacity-optimized-prioritized' | 'prioritized'; + EC2_INSTANCE_ARN_PREFIX: string | undefined; SCALE_ERRORS: string; } } diff --git a/lambdas/libs/compute-providers/core/index.ts b/lambdas/libs/compute-providers/core/index.ts index 64fc98e348..97274e14e1 100644 --- a/lambdas/libs/compute-providers/core/index.ts +++ b/lambdas/libs/compute-providers/core/index.ts @@ -31,6 +31,10 @@ export interface GitHubRunnerMetadata { export interface StartRunnerConfigOptions { runnerConfigStore?: import('@aws-github-runner/storage-providers').RunnerConfigStore; runnerGroupCacheStore?: import('@aws-github-runner/storage-providers').RunnerGroupCacheStore; + /** Compute provider that owns the runner IDs. Used by provider-neutral runner inventory. */ + computeProvider?: ComputeProviderType; + /** Access scope used by secret stores to isolate one runner's bootstrap configuration. */ + getRunnerConfigAccessScope?: (runnerId: string) => string | undefined; getRunnerConfigMetadata?: (runnerId: string) => { key: string; value: string }[]; onJitConfigCreated?: (runnerId: string, metadata: GitHubRunnerMetadata) => Promise; } @@ -76,7 +80,8 @@ export interface CreateRunnerResult { export interface ScaleUpComputeProvider extends ComputeProvider { resolveLabelsForRunners(messageLabels: string[]): Promise>; - getCurrentRunners(state: TState, input: CurrentRunnersInput): Promise; + /** Compute resource IDs currently owned by this runner entry. */ + getCurrentRunners(state: TState, input: CurrentRunnersInput): Promise; createRunners(input: CreateScaleUpRunnersInput): Promise; } @@ -125,7 +130,7 @@ export interface CreatePoolRunnersInput { storage?: import('@aws-github-runner/storage-providers').RunnerConfigStorage; } -export interface PoolComputeProvider extends ComputeProvider { +export interface PoolComputeProvider extends ComputeProvider { listRunners(input: ListPoolRunnersInput): Promise; countAvailableRunners( runners: TRunner[], diff --git a/lambdas/libs/storage-providers/aws/dynamodb/client.ts b/lambdas/libs/storage-providers/aws/dynamodb/client.ts new file mode 100644 index 0000000000..86493cc23f --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/client.ts @@ -0,0 +1,21 @@ +import { getTracedAWSV3Client } from '@aws-github-runner/aws-powertools-util'; +import { DynamoDBClient } from '@aws-sdk/client-dynamodb'; + +let memoisedClient: DynamoDBClient | undefined; + +export function getDynamoDbClient(): DynamoDBClient { + memoisedClient ??= getTracedAWSV3Client( + new DynamoDBClient({ + region: process.env.AWS_REGION, + maxAttempts: 10, + // One client serves two tables, so avoid an adaptive rate bucket coupling their throttling behavior. + retryMode: 'standard', + }), + ); + return memoisedClient; +} + +// Test-only reset for cases that need a fresh AWS SDK client. +export function resetDynamoDbClient(): void { + memoisedClient = undefined; +} diff --git a/lambdas/libs/storage-providers/aws/dynamodb/durable-config.ts b/lambdas/libs/storage-providers/aws/dynamodb/durable-config.ts new file mode 100644 index 0000000000..f22fde3269 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/durable-config.ts @@ -0,0 +1,37 @@ +import { GetItemCommand } from '@aws-sdk/client-dynamodb'; + +import { getDynamoDbClient } from './client'; +import { ID_ATTRIBUTE, SCOPE_ATTRIBUTE, VALUE_ATTRIBUTE } from './keys'; + +export async function getDurableConfigValue( + tableName: string, + scope: string, + id: string, + description: string, +): Promise { + const result = await getDynamoDbClient().send( + new GetItemCommand({ + TableName: tableName, + Key: { + [SCOPE_ATTRIBUTE]: { S: scope }, + [ID_ATTRIBUTE]: { S: id }, + }, + ConsistentRead: true, + ProjectionExpression: '#value', + ExpressionAttributeNames: { + '#value': VALUE_ATTRIBUTE, + }, + }), + ); + + if (!result.Item) { + throw new Error(`${description} item '${scope}/${id}' was not found`); + } + + const value = result.Item[VALUE_ATTRIBUTE]?.S; + if (value === undefined) { + throw new Error(`${description} item '${scope}/${id}' does not contain a string value`); + } + + return value; +} diff --git a/lambdas/libs/storage-providers/aws/dynamodb/environment.d.ts b/lambdas/libs/storage-providers/aws/dynamodb/environment.d.ts new file mode 100644 index 0000000000..59be4dbf10 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/environment.d.ts @@ -0,0 +1,13 @@ +export {}; + +declare global { + namespace NodeJS { + interface ProcessEnv { + RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME?: string; + RUNNER_CONFIG_DYNAMODB_ENTRY_ID?: string; + RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TABLE_NAME?: string; + RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TTL_SECONDS?: string; + RUNNER_CONFIG_DYNAMODB_TTL_SECONDS?: string; + } + } +} diff --git a/lambdas/libs/storage-providers/aws/dynamodb/environment.ts b/lambdas/libs/storage-providers/aws/dynamodb/environment.ts new file mode 100644 index 0000000000..fc0a394139 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/environment.ts @@ -0,0 +1,29 @@ +type DynamoDbEnvironmentVariable = + | 'RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME' + | 'RUNNER_CONFIG_DYNAMODB_ENTRY_ID' + | 'RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TABLE_NAME' + | 'RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TTL_SECONDS' + | 'RUNNER_CONFIG_DYNAMODB_TTL_SECONDS'; + +export function requiredEnvironmentValue(name: DynamoDbEnvironmentVariable): string { + const value = process.env[name]?.trim(); + if (!value) { + throw new Error(`Environment variable ${name} is not set`); + } + + return value; +} + +export function positiveIntegerEnvironmentValue(name: DynamoDbEnvironmentVariable): number { + const value = requiredEnvironmentValue(name); + if (!/^[1-9]\d*$/.test(value)) { + throw new Error(`Environment variable ${name} must be a positive integer`); + } + + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) { + throw new Error(`Environment variable ${name} must be a positive integer`); + } + + return parsed; +} diff --git a/lambdas/libs/storage-providers/aws/dynamodb/github-app-credentials-store.test.ts b/lambdas/libs/storage-providers/aws/dynamodb/github-app-credentials-store.test.ts new file mode 100644 index 0000000000..500ad90749 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/github-app-credentials-store.test.ts @@ -0,0 +1,103 @@ +import { DynamoDBClient, GetItemCommand } from '@aws-sdk/client-dynamodb'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest/vitest'; +import { beforeEach, describe, expect, it } from 'vitest'; + +import { resetDynamoDbClient } from './client'; +import { createAwsDynamoDbGitHubAppCredentialsStore } from './github-app-credentials-store'; + +const mockDynamoDbClient = mockClient(DynamoDBClient); +const cleanEnv = process.env; + +describe('aws_dynamodb GitHub App credentials store', () => { + beforeEach(() => { + mockDynamoDbClient.reset(); + resetDynamoDbClient(); + process.env = { ...cleanEnv }; + process.env.AWS_REGION = 'eu-west-1'; + process.env.RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME = 'runner-configuration'; + }); + + it('strongly reads and decodes an ordered credential array', async () => { + const value = JSON.stringify([ + { appId: 123, privateKeyBase64: Buffer.from('primary\\nkey').toString('base64') }, + { + appId: 456, + privateKeyBase64: Buffer.from('additional-key').toString('base64'), + installationId: 789, + }, + ]); + mockDynamoDbClient.on(GetItemCommand).resolves({ Item: { value: { S: value } } }); + const store = createAwsDynamoDbGitHubAppCredentialsStore(); + + await expect(store.get()).resolves.toEqual([ + { appId: 123, privateKey: 'primary\nkey', installationId: undefined }, + { appId: 456, privateKey: 'additional-key', installationId: 789 }, + ]); + expect(mockDynamoDbClient).toHaveReceivedCommandWith(GetItemCommand, { + TableName: 'runner-configuration', + Key: { + scope: { S: 'global#github-app' }, + id: { S: 'github-app-credentials' }, + }, + ConsistentRead: true, + ProjectionExpression: '#value', + ExpressionAttributeNames: { '#value': 'value' }, + }); + }); + + it.each([ + ['not-json', 'contains invalid JSON'], + ['[]', 'must contain a non-empty array'], + [JSON.stringify([null]), 'credential at index 0 has an invalid stored value'], + [JSON.stringify([{ appId: 0, privateKeyBase64: 'a2V5' }]), 'credential at index 0 has an invalid stored value'], + [ + JSON.stringify([{ appId: 1, privateKeyBase64: 'not-base64' }]), + 'credential at index 0 has an invalid stored value', + ], + [ + JSON.stringify([{ appId: 1, privateKeyBase64: 'a2V5', installationId: 1.5 }]), + 'credential at index 0 has an invalid stored value', + ], + ])('rejects malformed stored credentials without returning their value', async (value, message) => { + mockDynamoDbClient.on(GetItemCommand).resolves({ Item: { value: { S: value } } }); + const store = createAwsDynamoDbGitHubAppCredentialsStore(); + + await expect(store.get()).rejects.toThrow(message); + }); + + it('rejects a missing credentials item', async () => { + mockDynamoDbClient.on(GetItemCommand).resolves({}); + + await expect(createAwsDynamoDbGitHubAppCredentialsStore().get()).rejects.toThrow( + "GitHub App credentials item 'global#github-app/github-app-credentials' was not found", + ); + }); + + it('rejects a non-string credentials value', async () => { + mockDynamoDbClient.on(GetItemCommand).resolves({ Item: { value: { L: [] } } }); + + await expect(createAwsDynamoDbGitHubAppCredentialsStore().get()).rejects.toThrow( + "GitHub App credentials item 'global#github-app/github-app-credentials' does not contain a string value", + ); + }); + + it.each([undefined, '', ' '])('requires the durable table name for input %j', (tableName) => { + if (tableName === undefined) { + delete process.env.RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME; + } else { + process.env.RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME = tableName; + } + + expect(() => createAwsDynamoDbGitHubAppCredentialsStore()).toThrow( + 'Environment variable RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME is not set', + ); + }); + + it('propagates reads errors without exposing stored credentials', async () => { + const error = new Error('access denied'); + mockDynamoDbClient.on(GetItemCommand).rejects(error); + + await expect(createAwsDynamoDbGitHubAppCredentialsStore().get()).rejects.toBe(error); + }); +}); diff --git a/lambdas/libs/storage-providers/aws/dynamodb/github-app-credentials-store.ts b/lambdas/libs/storage-providers/aws/dynamodb/github-app-credentials-store.ts new file mode 100644 index 0000000000..e9b1a56c18 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/github-app-credentials-store.ts @@ -0,0 +1,89 @@ +import type { GitHubAppCredential, GitHubAppCredentialsStore } from '../../core'; +import { getDurableConfigValue } from './durable-config'; +import { requiredEnvironmentValue } from './environment'; +import { GITHUB_APP_CREDENTIALS_ID, GITHUB_APP_SCOPE } from './keys'; + +interface StoredGitHubAppCredential { + appId: number; + privateKeyBase64: string; + installationId?: number; +} + +export function createAwsDynamoDbGitHubAppCredentialsStore(): GitHubAppCredentialsStore { + return new AwsDynamoDbGitHubAppCredentialsStore(requiredEnvironmentValue('RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME')); +} + +class AwsDynamoDbGitHubAppCredentialsStore implements GitHubAppCredentialsStore { + constructor(private readonly tableName: string) {} + + async get(): Promise { + const value = await getDurableConfigValue( + this.tableName, + GITHUB_APP_SCOPE, + GITHUB_APP_CREDENTIALS_ID, + 'GitHub App credentials', + ); + const credentials = parseCredentials(value); + + return credentials.map((credential) => ({ + appId: credential.appId, + privateKey: decodePrivateKey(credential.privateKeyBase64), + installationId: credential.installationId, + })); + } +} + +function parseCredentials(value: string): StoredGitHubAppCredential[] { + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + throw new Error('GitHub App credentials item contains invalid JSON'); + } + + if (!Array.isArray(parsed) || parsed.length === 0) { + throw new Error('GitHub App credentials item must contain a non-empty array'); + } + + return parsed.map((credential, index) => parseCredential(credential, index)); +} + +function parseCredential(value: unknown, index: number): StoredGitHubAppCredential { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw invalidCredential(index); + } + + const credential = value as Record; + if (!isPositiveSafeInteger(credential.appId) || !isValidBase64(credential.privateKeyBase64)) { + throw invalidCredential(index); + } + if (credential.installationId !== undefined && !isPositiveSafeInteger(credential.installationId)) { + throw invalidCredential(index); + } + + return { + appId: credential.appId, + privateKeyBase64: credential.privateKeyBase64, + installationId: credential.installationId, + }; +} + +function isPositiveSafeInteger(value: unknown): value is number { + return typeof value === 'number' && Number.isSafeInteger(value) && value > 0; +} + +function isValidBase64(value: unknown): value is string { + if (typeof value !== 'string' || value.length === 0 || value.length % 4 !== 0) { + return false; + } + + return /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value); +} + +function decodePrivateKey(privateKeyBase64: string): string { + return Buffer.from(privateKeyBase64, 'base64').toString().replace(/\\n/g, '\n'); +} + +function invalidCredential(index: number): Error { + return new Error(`GitHub App credential at index ${index} has an invalid stored value`); +} diff --git a/lambdas/libs/storage-providers/aws/dynamodb/github-webhook-secret-store.test.ts b/lambdas/libs/storage-providers/aws/dynamodb/github-webhook-secret-store.test.ts new file mode 100644 index 0000000000..c66272c2c7 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/github-webhook-secret-store.test.ts @@ -0,0 +1,74 @@ +import { DynamoDBClient, GetItemCommand } from '@aws-sdk/client-dynamodb'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest/vitest'; +import { beforeEach, describe, expect, it } from 'vitest'; + +import { resetDynamoDbClient } from './client'; +import { createAwsDynamoDbGitHubWebhookSecretStore } from './github-webhook-secret-store'; + +const mockDynamoDbClient = mockClient(DynamoDBClient); +const cleanEnv = process.env; + +describe('aws_dynamodb GitHub webhook secret store', () => { + beforeEach(() => { + mockDynamoDbClient.reset(); + resetDynamoDbClient(); + process.env = { ...cleanEnv }; + process.env.AWS_REGION = 'eu-west-1'; + process.env.RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME = 'runner-configuration'; + }); + + it('strongly reads the global webhook secret', async () => { + mockDynamoDbClient.on(GetItemCommand).resolves({ Item: { value: { S: 'webhook-secret' } } }); + + await expect(createAwsDynamoDbGitHubWebhookSecretStore().get()).resolves.toBe('webhook-secret'); + expect(mockDynamoDbClient).toHaveReceivedCommandWith(GetItemCommand, { + TableName: 'runner-configuration', + Key: { + scope: { S: 'global#webhook' }, + id: { S: 'github-webhook-secret' }, + }, + ConsistentRead: true, + ProjectionExpression: '#value', + ExpressionAttributeNames: { '#value': 'value' }, + }); + }); + + it('leaves empty-value validation to the webhook config loader', async () => { + mockDynamoDbClient.on(GetItemCommand).resolves({ Item: { value: { S: '' } } }); + + await expect(createAwsDynamoDbGitHubWebhookSecretStore().get()).resolves.toBe(''); + }); + + it('rejects a missing secret item without logging or returning a value', async () => { + mockDynamoDbClient.on(GetItemCommand).resolves({}); + + await expect(createAwsDynamoDbGitHubWebhookSecretStore().get()).rejects.toThrow( + "GitHub webhook secret item 'global#webhook/github-webhook-secret' was not found", + ); + }); + + it('rejects a non-string secret item', async () => { + mockDynamoDbClient.on(GetItemCommand).resolves({ Item: { value: { B: new Uint8Array() } } }); + + await expect(createAwsDynamoDbGitHubWebhookSecretStore().get()).rejects.toThrow( + "GitHub webhook secret item 'global#webhook/github-webhook-secret' does not contain a string value", + ); + }); + + it('requires the durable table name before reading', () => { + delete process.env.RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME; + + expect(() => createAwsDynamoDbGitHubWebhookSecretStore()).toThrow( + 'Environment variable RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME is not set', + ); + expect(mockDynamoDbClient.calls()).toHaveLength(0); + }); + + it('propagates read errors without handling the secret value', async () => { + const error = new Error('access denied'); + mockDynamoDbClient.on(GetItemCommand).rejects(error); + + await expect(createAwsDynamoDbGitHubWebhookSecretStore().get()).rejects.toBe(error); + }); +}); diff --git a/lambdas/libs/storage-providers/aws/dynamodb/github-webhook-secret-store.ts b/lambdas/libs/storage-providers/aws/dynamodb/github-webhook-secret-store.ts new file mode 100644 index 0000000000..6fbf8f81b4 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/github-webhook-secret-store.ts @@ -0,0 +1,21 @@ +import type { GitHubWebhookSecretStore } from '../../core'; +import { getDurableConfigValue } from './durable-config'; +import { requiredEnvironmentValue } from './environment'; +import { GITHUB_WEBHOOK_SCOPE, GITHUB_WEBHOOK_SECRET_ID } from './keys'; + +export function createAwsDynamoDbGitHubWebhookSecretStore(): GitHubWebhookSecretStore { + return new AwsDynamoDbGitHubWebhookSecretStore(requiredEnvironmentValue('RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME')); +} + +class AwsDynamoDbGitHubWebhookSecretStore implements GitHubWebhookSecretStore { + constructor(private readonly tableName: string) {} + + async get(): Promise { + return await getDurableConfigValue( + this.tableName, + GITHUB_WEBHOOK_SCOPE, + GITHUB_WEBHOOK_SECRET_ID, + 'GitHub webhook secret', + ); + } +} diff --git a/lambdas/libs/storage-providers/aws/dynamodb/keys.test.ts b/lambdas/libs/storage-providers/aws/dynamodb/keys.test.ts new file mode 100644 index 0000000000..ac651df069 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/keys.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from 'vitest'; + +import { + GITHUB_APP_CREDENTIALS_ID, + GITHUB_APP_SCOPE, + GITHUB_WEBHOOK_SCOPE, + GITHUB_WEBHOOK_SECRET_ID, + RUNNER_BOOTSTRAP_CONFIG_ID, + RUNNER_CONFIG_ID, + RUNNER_MATCHER_CONFIG_ID, + RUNNER_MATCHER_SCOPE, + runnerBootstrapScope, + runnerGroupId, + runnerGroupScope, + runnerStateId, + runnerStateScope, +} from './keys'; + +describe('aws_dynamodb storage keys', () => { + it('isolates whole-deployment durable records by capability', () => { + expect({ scope: GITHUB_APP_SCOPE, id: GITHUB_APP_CREDENTIALS_ID }).toEqual({ + scope: 'global#github-app', + id: 'github-app-credentials', + }); + expect({ scope: GITHUB_WEBHOOK_SCOPE, id: GITHUB_WEBHOOK_SECRET_ID }).toEqual({ + scope: 'global#webhook', + id: 'github-webhook-secret', + }); + expect({ scope: RUNNER_MATCHER_SCOPE, id: RUNNER_MATCHER_CONFIG_ID }).toEqual({ + scope: 'global#matcher', + id: 'runner-matcher-config', + }); + }); + + it('isolates entry records by access boundary', () => { + expect({ scope: runnerBootstrapScope('linux-x64'), id: RUNNER_BOOTSTRAP_CONFIG_ID }).toEqual({ + scope: 'entry#linux-x64#bootstrap', + id: 'runner-config', + }); + expect({ scope: runnerGroupScope('linux-x64'), id: runnerGroupId('Default') }).toEqual({ + scope: 'entry#linux-x64#runner-group', + id: 'runner-group#Default', + }); + expect(RUNNER_CONFIG_ID).toBe('config'); + expect({ scope: runnerStateScope('linux-x64'), id: runnerStateId('runner-123') }).toEqual({ + scope: 'entry#linux-x64#runner-state', + id: 'runner#runner-123', + }); + }); +}); diff --git a/lambdas/libs/storage-providers/aws/dynamodb/keys.ts b/lambdas/libs/storage-providers/aws/dynamodb/keys.ts new file mode 100644 index 0000000000..cef92af6bb --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/keys.ts @@ -0,0 +1,34 @@ +export const SCOPE_ATTRIBUTE = 'scope'; +export const ID_ATTRIBUTE = 'id'; +export const VALUE_ATTRIBUTE = 'value'; +export const EXPIRES_AT_ATTRIBUTE = 'expires_at'; + +export const GITHUB_APP_SCOPE = 'global#github-app'; +export const GITHUB_WEBHOOK_SCOPE = 'global#webhook'; +export const RUNNER_MATCHER_SCOPE = 'global#matcher'; + +export const GITHUB_APP_CREDENTIALS_ID = 'github-app-credentials'; +export const GITHUB_WEBHOOK_SECRET_ID = 'github-webhook-secret'; +export const RUNNER_MATCHER_CONFIG_ID = 'runner-matcher-config'; +export const RUNNER_BOOTSTRAP_CONFIG_ID = 'runner-config'; +export const RUNNER_CONFIG_ID = 'config'; + +export function runnerBootstrapScope(entryId: string): string { + return `entry#${entryId}#bootstrap`; +} + +export function runnerGroupScope(entryId: string): string { + return `entry#${entryId}#runner-group`; +} + +export function runnerStateScope(entryId: string): string { + return `entry#${entryId}#runner-state`; +} + +export function runnerStateId(runnerId: string): string { + return `runner#${runnerId}`; +} + +export function runnerGroupId(runnerGroupName: string): string { + return `runner-group#${runnerGroupName}`; +} diff --git a/lambdas/libs/storage-providers/aws/dynamodb/runner-config-store.test.ts b/lambdas/libs/storage-providers/aws/dynamodb/runner-config-store.test.ts new file mode 100644 index 0000000000..1afb6c7fe9 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/runner-config-store.test.ts @@ -0,0 +1,148 @@ +import { DynamoDBClient, PutItemCommand } from '@aws-sdk/client-dynamodb'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest/vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { resetDynamoDbClient } from './client'; +import { createAwsDynamoDbRunnerConfigStore } from './runner-config-store'; + +const mockDynamoDbClient = mockClient(DynamoDBClient); +const cleanEnv = process.env; + +describe('aws_dynamodb runner config store', () => { + beforeEach(() => { + mockDynamoDbClient.reset(); + resetDynamoDbClient(); + process.env = { ...cleanEnv }; + process.env.AWS_REGION = 'eu-west-1'; + process.env.RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TABLE_NAME = 'runner-state'; + delete process.env.RUNNER_CONFIG_DYNAMODB_ENTRY_ID; + process.env.RUNNER_CONFIG_DYNAMODB_TTL_SECONDS = '3600'; + vi.useFakeTimers(); + vi.setSystemTime(new Date('2025-01-01T00:00:00.000Z')); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it('creates an expiring runner config without overwriting an existing record', async () => { + const store = createAwsDynamoDbRunnerConfigStore(); + + await store.create({ + runnerId: 'runner-123', + value: 'encoded-jit-config', + accessScope: 'arn:aws:ec2:eu-west-1:123456789012:instance/i-123', + }); + + expect(store.maxWritesPerSecond).toBeUndefined(); + expect(mockDynamoDbClient).toHaveReceivedCommandWith(PutItemCommand, { + TableName: 'runner-state', + Item: { + scope: { S: 'arn:aws:ec2:eu-west-1:123456789012:instance/i-123' }, + id: { S: 'config' }, + value: { S: 'encoded-jit-config' }, + expires_at: { N: '1735693200' }, + }, + ConditionExpression: 'attribute_not_exists(#scope) AND attribute_not_exists(#id)', + ExpressionAttributeNames: { + '#scope': 'scope', + '#id': 'id', + }, + }); + }); + + it('stores provider-neutral metadata without exposing it as top-level attributes', async () => { + const store = createAwsDynamoDbRunnerConfigStore(); + + await store.create( + { + runnerId: 'runner-123', + value: 'registration-config', + accessScope: 'arn:aws:ec2:eu-west-1:123456789012:instance/i-123', + }, + { + metadata: [ + { key: 'InstanceId', value: 'i-123' }, + { key: 'Environment', value: 'test' }, + ], + }, + ); + + const command = mockDynamoDbClient.commandCalls(PutItemCommand)[0].args[0]; + expect(command.input.Item?.metadata).toEqual({ + L: [ + { M: { key: { S: 'InstanceId' }, value: { S: 'i-123' } } }, + { M: { key: { S: 'Environment' }, value: { S: 'test' } } }, + ], + }); + }); + + it('does not write metadata for an empty metadata list', async () => { + const store = createAwsDynamoDbRunnerConfigStore(); + + await store.create( + { + runnerId: 'runner-123', + value: 'registration-config', + accessScope: 'arn:aws:ec2:eu-west-1:123456789012:instance/i-123', + }, + { metadata: [] }, + ); + + const command = mockDynamoDbClient.commandCalls(PutItemCommand)[0].args[0]; + expect(command.input.Item).not.toHaveProperty('metadata'); + }); + + it('relies on DynamoDB TTL instead of scanning or deleting during housekeeping', async () => { + const store = createAwsDynamoDbRunnerConfigStore(); + + await expect(store.houseKeeper()).resolves.toBeUndefined(); + + expect(mockDynamoDbClient.calls()).toHaveLength(0); + }); + + it.each(['RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TABLE_NAME', 'RUNNER_CONFIG_DYNAMODB_TTL_SECONDS'] as const)( + 'rejects a missing or blank %s', + (name) => { + delete process.env[name]; + expect(() => createAwsDynamoDbRunnerConfigStore()).toThrow(`Environment variable ${name} is not set`); + + process.env[name] = ' '; + expect(() => createAwsDynamoDbRunnerConfigStore()).toThrow(`Environment variable ${name} is not set`); + expect(mockDynamoDbClient.calls()).toHaveLength(0); + }, + ); + + it.each(['0', '-1', '1.5', 'not-a-number', '9007199254740992'])('rejects invalid TTL seconds %j', (ttlSeconds) => { + process.env.RUNNER_CONFIG_DYNAMODB_TTL_SECONDS = ttlSeconds; + + expect(() => createAwsDynamoDbRunnerConfigStore()).toThrow( + 'Environment variable RUNNER_CONFIG_DYNAMODB_TTL_SECONDS must be a positive integer', + ); + expect(mockDynamoDbClient.calls()).toHaveLength(0); + }); + + it.each([undefined, '', ' '])('rejects invalid access scope %j before writing', async (accessScope) => { + const store = createAwsDynamoDbRunnerConfigStore(); + + await expect(store.create({ runnerId: 'runner-123', value: 'sensitive-config', accessScope })).rejects.toThrow( + "Runner config field 'accessScope' must be a non-empty string for aws_dynamodb", + ); + expect(mockDynamoDbClient.calls()).toHaveLength(0); + }); + + it('propagates DynamoDB write errors without handling the stored value', async () => { + const error = new Error('conditional request failed'); + mockDynamoDbClient.on(PutItemCommand).rejects(error); + const store = createAwsDynamoDbRunnerConfigStore(); + + await expect( + store.create({ + runnerId: 'runner-123', + value: 'sensitive-config', + accessScope: 'arn:aws:ec2:eu-west-1:123456789012:instance/i-123', + }), + ).rejects.toBe(error); + }); +}); diff --git a/lambdas/libs/storage-providers/aws/dynamodb/runner-config-store.ts b/lambdas/libs/storage-providers/aws/dynamodb/runner-config-store.ts new file mode 100644 index 0000000000..79de2b6b71 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/runner-config-store.ts @@ -0,0 +1,66 @@ +import { PutItemCommand, type AttributeValue } from '@aws-sdk/client-dynamodb'; + +import type { RunnerConfigMetadata, RunnerConfigRecord, RunnerConfigStore } from '../../core'; +import { getDynamoDbClient } from './client'; +import { positiveIntegerEnvironmentValue, requiredEnvironmentValue } from './environment'; +import { EXPIRES_AT_ATTRIBUTE, ID_ATTRIBUTE, RUNNER_CONFIG_ID, SCOPE_ATTRIBUTE, VALUE_ATTRIBUTE } from './keys'; + +const METADATA_ATTRIBUTE = 'metadata'; + +interface AwsDynamoDbRunnerConfigStoreConfig { + tableName: string; + ttlSeconds: number; +} + +export function createAwsDynamoDbRunnerConfigStore(): RunnerConfigStore { + return new AwsDynamoDbRunnerConfigStore({ + tableName: requiredEnvironmentValue('RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TABLE_NAME'), + ttlSeconds: positiveIntegerEnvironmentValue('RUNNER_CONFIG_DYNAMODB_TTL_SECONDS'), + }); +} + +class AwsDynamoDbRunnerConfigStore implements RunnerConfigStore { + constructor(private readonly config: AwsDynamoDbRunnerConfigStoreConfig) {} + + async create(record: RunnerConfigRecord, options: { metadata?: RunnerConfigMetadata[] } = {}): Promise { + if (typeof record.accessScope !== 'string' || record.accessScope.trim() === '') { + throw new Error("Runner config field 'accessScope' must be a non-empty string for aws_dynamodb"); + } + + const item: Record = { + [SCOPE_ATTRIBUTE]: { S: record.accessScope }, + [ID_ATTRIBUTE]: { S: RUNNER_CONFIG_ID }, + [VALUE_ATTRIBUTE]: { S: record.value }, + [EXPIRES_AT_ATTRIBUTE]: { + N: (Math.floor(Date.now() / 1000) + this.config.ttlSeconds).toString(), + }, + }; + + if (options.metadata && options.metadata.length > 0) { + item[METADATA_ATTRIBUTE] = { + L: options.metadata.map(({ key, value }) => ({ + M: { + key: { S: key }, + value: { S: value }, + }, + })), + }; + } + + await getDynamoDbClient().send( + new PutItemCommand({ + TableName: this.config.tableName, + Item: item, + ConditionExpression: 'attribute_not_exists(#scope) AND attribute_not_exists(#id)', + ExpressionAttributeNames: { + '#scope': SCOPE_ATTRIBUTE, + '#id': ID_ATTRIBUTE, + }, + }), + ); + } + + async houseKeeper(): Promise { + // DynamoDB TTL removes expired runner config records without a scan/delete job. + } +} diff --git a/lambdas/libs/storage-providers/aws/dynamodb/runner-group-cache-store.test.ts b/lambdas/libs/storage-providers/aws/dynamodb/runner-group-cache-store.test.ts new file mode 100644 index 0000000000..2fbac5b69d --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/runner-group-cache-store.test.ts @@ -0,0 +1,108 @@ +import { DynamoDBClient, GetItemCommand, PutItemCommand } from '@aws-sdk/client-dynamodb'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest/vitest'; +import { beforeEach, describe, expect, it } from 'vitest'; + +import { resetDynamoDbClient } from './client'; +import { createAwsDynamoDbRunnerGroupCacheStore } from './runner-group-cache-store'; + +const mockDynamoDbClient = mockClient(DynamoDBClient); +const cleanEnv = process.env; + +describe('aws_dynamodb runner group cache store', () => { + beforeEach(() => { + mockDynamoDbClient.reset(); + resetDynamoDbClient(); + process.env = { ...cleanEnv }; + process.env.AWS_REGION = 'eu-west-1'; + process.env.RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME = 'runner-configuration'; + process.env.RUNNER_CONFIG_DYNAMODB_ENTRY_ID = 'linux-x64'; + }); + + it('gets a runner group id with a strongly consistent projected read', async () => { + mockDynamoDbClient.on(GetItemCommand).resolves({ Item: { value: { S: '42' } } }); + const store = createAwsDynamoDbRunnerGroupCacheStore(); + + await expect(store.get('Default')).resolves.toBe(42); + expect(mockDynamoDbClient).toHaveReceivedCommandWith(GetItemCommand, { + TableName: 'runner-configuration', + Key: { + scope: { S: 'entry#linux-x64#runner-group' }, + id: { S: 'runner-group#Default' }, + }, + ConsistentRead: true, + ProjectionExpression: '#value', + ExpressionAttributeNames: { + '#value': 'value', + }, + }); + }); + + it('returns undefined when the runner group is not cached', async () => { + mockDynamoDbClient.on(GetItemCommand).resolves({}); + const store = createAwsDynamoDbRunnerGroupCacheStore(); + + await expect(store.get('Default')).resolves.toBeUndefined(); + }); + + it.each([ + [{ value: { N: '42' } }, 'non-string value'], + [{ value: { S: '42cached' } }, 'partially numeric value'], + [{ value: { S: '9007199254740992' } }, 'unsafe integer value'], + ])('rejects an invalid cached runner group id: %s', async (item) => { + mockDynamoDbClient.on(GetItemCommand).resolves({ Item: item }); + const store = createAwsDynamoDbRunnerGroupCacheStore(); + + await expect(store.get('Default')).rejects.toThrow( + "Runner group cache item 'entry#linux-x64#runner-group/runner-group#Default' has an invalid value", + ); + }); + + it('creates a runner group cache record without overwriting an existing record', async () => { + const store = createAwsDynamoDbRunnerGroupCacheStore(); + + await store.create({ runnerGroupName: 'Default', runnerGroupId: 42 }); + + expect(mockDynamoDbClient).toHaveReceivedCommandWith(PutItemCommand, { + TableName: 'runner-configuration', + Item: { + scope: { S: 'entry#linux-x64#runner-group' }, + id: { S: 'runner-group#Default' }, + value: { S: '42' }, + }, + ConditionExpression: 'attribute_not_exists(#scope) AND attribute_not_exists(#id)', + ExpressionAttributeNames: { + '#scope': 'scope', + '#id': 'id', + }, + }); + }); + + it.each(['RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME', 'RUNNER_CONFIG_DYNAMODB_ENTRY_ID'] as const)( + 'rejects a missing or blank %s', + (name) => { + delete process.env[name]; + expect(() => createAwsDynamoDbRunnerGroupCacheStore()).toThrow(`Environment variable ${name} is not set`); + + process.env[name] = ' '; + expect(() => createAwsDynamoDbRunnerGroupCacheStore()).toThrow(`Environment variable ${name} is not set`); + expect(mockDynamoDbClient.calls()).toHaveLength(0); + }, + ); + + it('propagates DynamoDB read errors', async () => { + const error = new Error('read failed'); + mockDynamoDbClient.on(GetItemCommand).rejects(error); + const store = createAwsDynamoDbRunnerGroupCacheStore(); + + await expect(store.get('Default')).rejects.toBe(error); + }); + + it('propagates DynamoDB write errors', async () => { + const error = new Error('conditional request failed'); + mockDynamoDbClient.on(PutItemCommand).rejects(error); + const store = createAwsDynamoDbRunnerGroupCacheStore(); + + await expect(store.create({ runnerGroupName: 'Default', runnerGroupId: 42 })).rejects.toBe(error); + }); +}); diff --git a/lambdas/libs/storage-providers/aws/dynamodb/runner-group-cache-store.ts b/lambdas/libs/storage-providers/aws/dynamodb/runner-group-cache-store.ts new file mode 100644 index 0000000000..a90a79e46e --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/runner-group-cache-store.ts @@ -0,0 +1,80 @@ +import { GetItemCommand, PutItemCommand } from '@aws-sdk/client-dynamodb'; + +import type { RunnerGroupCacheRecord, RunnerGroupCacheStore } from '../../core'; +import { getDynamoDbClient } from './client'; +import { requiredEnvironmentValue } from './environment'; +import { + ID_ATTRIBUTE, + runnerGroupId as runnerGroupItemId, + runnerGroupScope, + SCOPE_ATTRIBUTE, + VALUE_ATTRIBUTE, +} from './keys'; + +interface AwsDynamoDbRunnerGroupCacheStoreConfig { + tableName: string; + scope: string; +} + +export function createAwsDynamoDbRunnerGroupCacheStore(): RunnerGroupCacheStore { + return new AwsDynamoDbRunnerGroupCacheStore({ + tableName: requiredEnvironmentValue('RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME'), + scope: runnerGroupScope(requiredEnvironmentValue('RUNNER_CONFIG_DYNAMODB_ENTRY_ID')), + }); +} + +class AwsDynamoDbRunnerGroupCacheStore implements RunnerGroupCacheStore { + constructor(private readonly config: AwsDynamoDbRunnerGroupCacheStoreConfig) {} + + async get(runnerGroupName: string): Promise { + const id = runnerGroupItemId(runnerGroupName); + const result = await getDynamoDbClient().send( + new GetItemCommand({ + TableName: this.config.tableName, + Key: { + [SCOPE_ATTRIBUTE]: { S: this.config.scope }, + [ID_ATTRIBUTE]: { S: id }, + }, + ConsistentRead: true, + ProjectionExpression: '#value', + ExpressionAttributeNames: { + '#value': VALUE_ATTRIBUTE, + }, + }), + ); + + if (!result.Item) { + return undefined; + } + + const value = result.Item[VALUE_ATTRIBUTE]?.S; + if (value === undefined || !/^\d+$/.test(value)) { + throw new Error(`Runner group cache item '${this.config.scope}/${id}' has an invalid value`); + } + + const runnerGroupId = Number(value); + if (!Number.isSafeInteger(runnerGroupId)) { + throw new Error(`Runner group cache item '${this.config.scope}/${id}' has an invalid value`); + } + + return runnerGroupId; + } + + async create(record: RunnerGroupCacheRecord): Promise { + await getDynamoDbClient().send( + new PutItemCommand({ + TableName: this.config.tableName, + Item: { + [SCOPE_ATTRIBUTE]: { S: this.config.scope }, + [ID_ATTRIBUTE]: { S: runnerGroupItemId(record.runnerGroupName) }, + [VALUE_ATTRIBUTE]: { S: record.runnerGroupId.toString() }, + }, + ConditionExpression: 'attribute_not_exists(#scope) AND attribute_not_exists(#id)', + ExpressionAttributeNames: { + '#scope': SCOPE_ATTRIBUTE, + '#id': ID_ATTRIBUTE, + }, + }), + ); + } +} diff --git a/lambdas/libs/storage-providers/aws/dynamodb/runner-matcher-config-store.test.ts b/lambdas/libs/storage-providers/aws/dynamodb/runner-matcher-config-store.test.ts new file mode 100644 index 0000000000..66e8d07c2e --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/runner-matcher-config-store.test.ts @@ -0,0 +1,95 @@ +import { DynamoDBClient, GetItemCommand } from '@aws-sdk/client-dynamodb'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest/vitest'; +import { beforeEach, describe, expect, it } from 'vitest'; + +import { resetDynamoDbClient } from './client'; +import { createAwsDynamoDbRunnerMatcherConfigStore } from './runner-matcher-config-store'; + +const mockDynamoDbClient = mockClient(DynamoDBClient); +const cleanEnv = process.env; + +describe('aws_dynamodb runner matcher config store', () => { + beforeEach(() => { + mockDynamoDbClient.reset(); + resetDynamoDbClient(); + process.env = { ...cleanEnv }; + process.env.AWS_REGION = 'eu-west-1'; + process.env.RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME = 'runner-configuration'; + }); + + it('gets the matcher config with a strongly consistent projected read', async () => { + mockDynamoDbClient.on(GetItemCommand).resolves({ Item: { value: { S: '[{"id":"runner"}]' } } }); + const store = createAwsDynamoDbRunnerMatcherConfigStore(); + + await expect(store.get()).resolves.toBe('[{"id":"runner"}]'); + expect(mockDynamoDbClient).toHaveReceivedCommandWith(GetItemCommand, { + TableName: 'runner-configuration', + Key: { + scope: { S: 'global#matcher' }, + id: { S: 'runner-matcher-config' }, + }, + ConsistentRead: true, + ProjectionExpression: '#value', + ExpressionAttributeNames: { + '#value': 'value', + }, + }); + }); + + it('returns an empty stored string for validation by the webhook config loader', async () => { + mockDynamoDbClient.on(GetItemCommand).resolves({ Item: { value: { S: '' } } }); + const store = createAwsDynamoDbRunnerMatcherConfigStore(); + + await expect(store.get()).resolves.toBe(''); + }); + + it('rejects a missing matcher config item', async () => { + mockDynamoDbClient.on(GetItemCommand).resolves({}); + const store = createAwsDynamoDbRunnerMatcherConfigStore(); + + await expect(store.get()).rejects.toThrow( + "Runner matcher config item 'global#matcher/runner-matcher-config' was not found", + ); + }); + + it('rejects a matcher config item without a string value', async () => { + mockDynamoDbClient.on(GetItemCommand).resolves({ Item: { value: { N: '1' } } }); + const store = createAwsDynamoDbRunnerMatcherConfigStore(); + + await expect(store.get()).rejects.toThrow( + "Runner matcher config item 'global#matcher/runner-matcher-config' does not contain a string value", + ); + }); + + it('rejects a missing or blank durable table name', () => { + delete process.env.RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME; + expect(() => createAwsDynamoDbRunnerMatcherConfigStore()).toThrow( + 'Environment variable RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME is not set', + ); + + process.env.RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME = ' '; + expect(() => createAwsDynamoDbRunnerMatcherConfigStore()).toThrow( + 'Environment variable RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME is not set', + ); + expect(mockDynamoDbClient.calls()).toHaveLength(0); + }); + + it('propagates DynamoDB read errors', async () => { + const error = new Error('read failed'); + mockDynamoDbClient.on(GetItemCommand).rejects(error); + const store = createAwsDynamoDbRunnerMatcherConfigStore(); + + await expect(store.get()).rejects.toBe(error); + }); + + it('reuses the memoised client across reads', async () => { + mockDynamoDbClient.on(GetItemCommand).resolves({ Item: { value: { S: '[]' } } }); + const store = createAwsDynamoDbRunnerMatcherConfigStore(); + + await store.get(); + await store.get(); + + expect(mockDynamoDbClient.commandCalls(GetItemCommand)).toHaveLength(2); + }); +}); diff --git a/lambdas/libs/storage-providers/aws/dynamodb/runner-matcher-config-store.ts b/lambdas/libs/storage-providers/aws/dynamodb/runner-matcher-config-store.ts new file mode 100644 index 0000000000..f840345983 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/runner-matcher-config-store.ts @@ -0,0 +1,27 @@ +import type { RunnerMatcherConfigStore } from '../../core'; +import { getDurableConfigValue } from './durable-config'; +import { requiredEnvironmentValue } from './environment'; +import { RUNNER_MATCHER_CONFIG_ID, RUNNER_MATCHER_SCOPE } from './keys'; + +interface AwsDynamoDbRunnerMatcherConfigStoreConfig { + tableName: string; +} + +export function createAwsDynamoDbRunnerMatcherConfigStore(): RunnerMatcherConfigStore { + return new AwsDynamoDbRunnerMatcherConfigStore({ + tableName: requiredEnvironmentValue('RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME'), + }); +} + +class AwsDynamoDbRunnerMatcherConfigStore implements RunnerMatcherConfigStore { + constructor(private readonly config: AwsDynamoDbRunnerMatcherConfigStoreConfig) {} + + async get(): Promise { + return await getDurableConfigValue( + this.config.tableName, + RUNNER_MATCHER_SCOPE, + RUNNER_MATCHER_CONFIG_ID, + 'Runner matcher config', + ); + } +} diff --git a/lambdas/libs/storage-providers/aws/dynamodb/runner-state-store.test.ts b/lambdas/libs/storage-providers/aws/dynamodb/runner-state-store.test.ts new file mode 100644 index 0000000000..75e74ab526 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/runner-state-store.test.ts @@ -0,0 +1,451 @@ +import { + ConditionalCheckFailedException, + DeleteItemCommand, + DynamoDBClient, + PutItemCommand, + QueryCommand, + UpdateItemCommand, + type AttributeValue, +} from '@aws-sdk/client-dynamodb'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest/vitest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { CreateRunnerStateRecord } from '../../core'; +import { resetDynamoDbClient } from './client'; +import { createAwsDynamoDbRunnerStateStore } from './runner-state-store'; + +const mockDynamoDbClient = mockClient(DynamoDBClient); +const cleanEnv = process.env; + +describe('aws_dynamodb runner state store', () => { + beforeEach(() => { + mockDynamoDbClient.reset(); + resetDynamoDbClient(); + process.env = { ...cleanEnv }; + process.env.AWS_REGION = 'eu-west-1'; + process.env.RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TABLE_NAME = 'runner-state'; + process.env.RUNNER_CONFIG_DYNAMODB_ENTRY_ID = 'linux-x64'; + process.env.RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TTL_SECONDS = '86400'; + mockDynamoDbClient.on(PutItemCommand).resolves({}); + mockDynamoDbClient.on(UpdateItemCommand).resolves({}); + mockDynamoDbClient.on(DeleteItemCommand).resolves({}); + vi.useFakeTimers(); + vi.setSystemTime(new Date('2025-01-01T00:00:00.000Z')); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it('creates a provisioning record without a secret payload or overwrite', async () => { + const store = createAwsDynamoDbRunnerStateStore(); + + await store.create(createRecord()); + + expect(mockDynamoDbClient).toHaveReceivedCommandWith(PutItemCommand, { + TableName: 'runner-state', + Item: { + scope: { S: 'entry#linux-x64#runner-state' }, + id: { S: 'runner#runner-123' }, + runner_id: { S: 'runner-123' }, + compute_provider: { S: 'aws_ec2' }, + compute_resource_id: { S: 'i-123' }, + runner_name: { S: 'ghr-runner-123' }, + runner_labels: { L: [{ S: 'linux' }, { S: 'x64' }] }, + runner_owner: { S: 'github-aws-runners' }, + runner_type: { S: 'Org' }, + state: { S: 'provisioning' }, + created_at: { S: '2025-01-01T00:00:00.000Z' }, + updated_at: { S: '2025-01-01T00:00:00.000Z' }, + expires_at: { N: '1735776000' }, + metadata: { + L: [{ M: { key: { S: 'Environment' }, value: { S: 'test' } } }], + }, + }, + ConditionExpression: 'attribute_not_exists(#scope) AND attribute_not_exists(#id)', + ExpressionAttributeNames: { '#scope': 'scope', '#id': 'id' }, + }); + const item = mockDynamoDbClient.commandCalls(PutItemCommand)[0].args[0].input.Item; + expect(item).not.toHaveProperty('value'); + }); + + it('omits optional attributes when provisioning data is not available yet', async () => { + const store = createAwsDynamoDbRunnerStateStore(); + const record = createRecord(); + delete record.runnerName; + delete record.runnerLabels; + delete record.metadata; + + await store.create(record); + + const item = mockDynamoDbClient.commandCalls(PutItemCommand)[0].args[0].input.Item; + expect(item).not.toHaveProperty('runner_name'); + expect(item).not.toHaveProperty('runner_labels'); + expect(item).not.toHaveProperty('metadata'); + }); + + it('preserves empty provider-neutral metadata values', async () => { + const store = createAwsDynamoDbRunnerStateStore(); + + await store.create({ ...createRecord(), metadata: [{ key: 'OptionalTag', value: '' }] }); + + expect(mockDynamoDbClient.commandCalls(PutItemCommand)[0].args[0].input.Item?.metadata).toEqual({ + L: [{ M: { key: { S: 'OptionalTag' }, value: { S: '' } } }], + }); + }); + + it('activates only provisioning records and atomically adds GitHub identity metadata', async () => { + const store = createAwsDynamoDbRunnerStateStore(); + + await store.activate('runner-123', { + githubRunnerId: '9876', + runnerName: 'jit-runner', + runnerLabels: ['linux', 'arm64'], + metadata: [{ key: 'zone', value: 'eu-west-1a' }], + }); + + expect(mockDynamoDbClient).toHaveReceivedCommandWith(UpdateItemCommand, { + TableName: 'runner-state', + Key: { scope: { S: 'entry#linux-x64#runner-state' }, id: { S: 'runner#runner-123' } }, + UpdateExpression: + 'SET #state = :state, #updatedAt = :updatedAt, #runnerName = :runnerName, #runnerLabels = :runnerLabels, #githubRunnerId = :githubRunnerId, #metadata = :metadata REMOVE #expiresAt', + ConditionExpression: 'attribute_exists(#scope) AND attribute_exists(#id) AND #state IN (:expectedState0)', + ExpressionAttributeNames: { + '#scope': 'scope', + '#id': 'id', + '#state': 'state', + '#updatedAt': 'updated_at', + '#expiresAt': 'expires_at', + '#runnerName': 'runner_name', + '#runnerLabels': 'runner_labels', + '#githubRunnerId': 'github_runner_id', + '#metadata': 'metadata', + }, + ExpressionAttributeValues: { + ':state': { S: 'active' }, + ':updatedAt': { S: '2025-01-01T00:00:00.000Z' }, + ':runnerName': { S: 'jit-runner' }, + ':runnerLabels': { L: [{ S: 'linux' }, { S: 'arm64' }] }, + ':githubRunnerId': { S: '9876' }, + ':metadata': { L: [{ M: { key: { S: 'zone' }, value: { S: 'eu-west-1a' } } }] }, + ':expectedState0': { S: 'provisioning' }, + }, + }); + }); + + it('records GitHub identity while the runner remains provisioning', async () => { + const store = createAwsDynamoDbRunnerStateStore(); + + await store.recordGitHubIdentity('runner-123', { + githubRunnerId: '9876', + runnerName: 'jit-runner', + runnerLabels: ['linux', 'arm64'], + }); + + expect(mockDynamoDbClient).toHaveReceivedCommandWith(UpdateItemCommand, { + TableName: 'runner-state', + Key: { scope: { S: 'entry#linux-x64#runner-state' }, id: { S: 'runner#runner-123' } }, + UpdateExpression: + 'SET #state = :state, #updatedAt = :updatedAt, #expiresAt = :expiresAt, #runnerName = :runnerName, #runnerLabels = :runnerLabels, #githubRunnerId = :githubRunnerId', + ConditionExpression: 'attribute_exists(#scope) AND attribute_exists(#id) AND #state IN (:expectedState0)', + ExpressionAttributeValues: expect.objectContaining({ + ':state': { S: 'provisioning' }, + ':updatedAt': { S: '2025-01-01T00:00:00.000Z' }, + ':expiresAt': { N: '1735776000' }, + ':runnerName': { S: 'jit-runner' }, + ':runnerLabels': { L: [{ S: 'linux' }, { S: 'arm64' }] }, + ':githubRunnerId': { S: '9876' }, + ':expectedState0': { S: 'provisioning' }, + }), + }); + }); + + it('lists all pages for an entry with a strongly consistent query', async () => { + const lastKey = { scope: { S: 'entry#linux-x64#runner-state' }, id: { S: 'runner#runner-123' } }; + mockDynamoDbClient + .on(QueryCommand) + .resolvesOnce({ Items: [storedRecord()], LastEvaluatedKey: lastKey }) + .resolvesOnce({ Items: [storedRecord({ runnerId: 'runner-456', resourceId: 'vm-456' })] }); + const store = createAwsDynamoDbRunnerStateStore(); + + await expect(store.list()).resolves.toEqual([ + expectedRecord(), + expectedRecord({ runnerId: 'runner-456', resourceId: 'vm-456' }), + ]); + const calls = mockDynamoDbClient.commandCalls(QueryCommand); + expect(calls).toHaveLength(2); + expect(calls[0].args[0].input).toMatchObject({ + TableName: 'runner-state', + KeyConditionExpression: '#scope = :scope AND begins_with(#id, :runner)', + ConsistentRead: true, + ExpressionAttributeValues: { + ':scope': { S: 'entry#linux-x64#runner-state' }, + ':runner': { S: 'runner#' }, + }, + }); + expect(calls[1].args[0].input.ExclusiveStartKey).toEqual(lastKey); + }); + + it('filters by compute provider while retaining an entry-scoped key query', async () => { + mockDynamoDbClient.on(QueryCommand).resolves({ Items: [] }); + const store = createAwsDynamoDbRunnerStateStore(); + + await store.list({ computeProvider: 'aws_microvm' }); + + expect(mockDynamoDbClient).toHaveReceivedCommandWith(QueryCommand, { + FilterExpression: '#computeProvider = :computeProvider', + ExpressionAttributeNames: { + '#scope': 'scope', + '#id': 'id', + '#computeProvider': 'compute_provider', + }, + ExpressionAttributeValues: { + ':scope': { S: 'entry#linux-x64#runner-state' }, + ':runner': { S: 'runner#' }, + ':computeProvider': { S: 'aws_microvm' }, + }, + }); + }); + + it('maps optional fields as absent and rejects corrupt lifecycle state', async () => { + const item = storedRecord(); + delete item.runner_name; + delete item.runner_labels; + delete item.github_runner_id; + delete item.metadata; + mockDynamoDbClient.on(QueryCommand).resolves({ Items: [item] }); + const store = createAwsDynamoDbRunnerStateStore(); + + await expect(store.list()).resolves.toEqual([ + expect.objectContaining({ + runnerName: undefined, + runnerLabels: undefined, + githubRunnerId: undefined, + metadata: undefined, + }), + ]); + + item.state = { S: 'unknown' }; + mockDynamoDbClient.on(QueryCommand).resolves({ Items: [item] }); + await expect(store.list()).rejects.toThrow( + "Runner state item 'entry#linux-x64#runner-state/runner#runner-123' has an invalid 'state' attribute", + ); + }); + + it('marks and unmarks orphan state with conditional transitions', async () => { + const store = createAwsDynamoDbRunnerStateStore(); + + await store.markOrphan('runner-123'); + await store.unmarkOrphan('runner-123'); + + const calls = mockDynamoDbClient.commandCalls(UpdateItemCommand); + expect(calls[0].args[0].input.ExpressionAttributeValues).toMatchObject({ + ':state': { S: 'orphan' }, + ':expectedState0': { S: 'active' }, + }); + expect(calls[0].args[0].input.UpdateExpression).toContain('REMOVE #expiresAt'); + expect(calls[0].args[0].input.ExpressionAttributeValues).not.toHaveProperty(':expiresAt'); + expect(calls[1].args[0].input.ExpressionAttributeValues).toMatchObject({ + ':state': { S: 'active' }, + ':expectedState0': { S: 'orphan' }, + }); + expect(calls[1].args[0].input.UpdateExpression).toContain('REMOVE #expiresAt'); + expect(calls[1].args[0].input.ExpressionAttributeValues).not.toHaveProperty(':expiresAt'); + }); + + it.each(['provisioning', 'active', 'orphan'] as const)( + 'claims termination and returns the prior %s state', + async (state) => { + mockDynamoDbClient.on(UpdateItemCommand).resolves({ Attributes: { state: { S: state } } }); + const store = createAwsDynamoDbRunnerStateStore(); + + await expect(store.beginTermination('runner-123')).resolves.toBe(state); + expect(mockDynamoDbClient).toHaveReceivedCommandWith(UpdateItemCommand, { + ReturnValues: 'ALL_OLD', + ConditionExpression: + 'attribute_exists(#scope) AND attribute_exists(#id) AND (#state IN (:provisioning, :active, :orphan) OR (#state = :terminating AND #updatedAt < :staleBefore))', + ExpressionAttributeValues: expect.objectContaining({ + ':terminating': { S: 'terminating' }, + ':expiresAt': { N: '1735776000' }, + ':provisioning': { S: 'provisioning' }, + ':active': { S: 'active' }, + ':orphan': { S: 'orphan' }, + ':staleBefore': { S: '2024-12-31T23:44:00.000Z' }, + }), + }); + }, + ); + + it('reclaims a stale terminating record on a later invocation without allowing an immediate double claim', async () => { + mockDynamoDbClient + .on(UpdateItemCommand) + .resolvesOnce({ Attributes: { state: { S: 'active' } } }) + .rejectsOnce(new ConditionalCheckFailedException({ $metadata: {}, message: 'lease is still held' })) + .resolvesOnce({ Attributes: { state: { S: 'terminating' } } }); + const store = createAwsDynamoDbRunnerStateStore(); + + await expect(store.beginTermination('runner-123')).resolves.toBe('active'); + await expect(store.beginTermination('runner-123')).resolves.toBeUndefined(); + vi.advanceTimersByTime(17 * 60 * 1000); + await expect(store.beginTermination('runner-123')).resolves.toBe('terminating'); + + const reclaimed = mockDynamoDbClient.commandCalls(UpdateItemCommand)[2].args[0].input; + expect(reclaimed.ExpressionAttributeValues?.[':staleBefore']).toEqual({ + S: '2025-01-01T00:01:00.000Z', + }); + }); + + it('returns undefined when another invocation already owns termination', async () => { + mockDynamoDbClient + .on(UpdateItemCommand) + .rejects(new ConditionalCheckFailedException({ $metadata: {}, message: 'condition failed' })); + const store = createAwsDynamoDbRunnerStateStore(); + + await expect(store.beginTermination('runner-123')).resolves.toBeUndefined(); + }); + + it('restores the safety TTL when cancellation returns a runner to provisioning', async () => { + const store = createAwsDynamoDbRunnerStateStore(); + + await store.cancelTermination('runner-123', 'provisioning'); + + expect(mockDynamoDbClient).toHaveReceivedCommandWith(UpdateItemCommand, { + UpdateExpression: 'SET #state = :state, #updatedAt = :updatedAt, #expiresAt = :expiresAt', + ExpressionAttributeValues: expect.objectContaining({ + ':state': { S: 'provisioning' }, + ':expiresAt': { N: '1735776000' }, + ':expectedState0': { S: 'terminating' }, + }), + }); + }); + + it.each(['active', 'orphan'] as const)('removes the safety TTL when cancellation restores %s', async (state) => { + const store = createAwsDynamoDbRunnerStateStore(); + + await store.cancelTermination('runner-123', state); + + expect(mockDynamoDbClient).toHaveReceivedCommandWith(UpdateItemCommand, { + UpdateExpression: 'SET #state = :state, #updatedAt = :updatedAt REMOVE #expiresAt', + ExpressionAttributeValues: expect.objectContaining({ + ':state': { S: state }, + ':expectedState0': { S: 'terminating' }, + }), + }); + const values = mockDynamoDbClient.commandCalls(UpdateItemCommand)[0].args[0].input.ExpressionAttributeValues; + expect(values).not.toHaveProperty(':expiresAt'); + }); + + it('deletes only a record whose termination was claimed', async () => { + const store = createAwsDynamoDbRunnerStateStore(); + + await store.delete('runner-123'); + + expect(mockDynamoDbClient).toHaveReceivedCommandWith(DeleteItemCommand, { + TableName: 'runner-state', + Key: { scope: { S: 'entry#linux-x64#runner-state' }, id: { S: 'runner#runner-123' } }, + ConditionExpression: '#state = :terminating', + ExpressionAttributeNames: { '#state': 'state' }, + ExpressionAttributeValues: { ':terminating': { S: 'terminating' } }, + }); + }); + + it.each([ + 'RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TABLE_NAME', + 'RUNNER_CONFIG_DYNAMODB_ENTRY_ID', + 'RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TTL_SECONDS', + ] as const)('rejects missing provider environment %s', (name) => { + delete process.env[name]; + + expect(() => createAwsDynamoDbRunnerStateStore()).toThrow(`Environment variable ${name} is not set`); + expect(mockDynamoDbClient.calls()).toHaveLength(0); + }); + + it.each(['0', '-1', '1.5', 'not-a-number'])('rejects invalid state TTL %j', (ttl) => { + process.env.RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TTL_SECONDS = ttl; + + expect(() => createAwsDynamoDbRunnerStateStore()).toThrow( + 'Environment variable RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TTL_SECONDS must be a positive integer', + ); + }); + + it('validates records before writing', async () => { + const store = createAwsDynamoDbRunnerStateStore(); + + await expect(store.create({ ...createRecord(), computeProvider: ' ' })).rejects.toThrow( + "Runner state field 'computeProvider' must be a non-empty string", + ); + await expect(store.create({ ...createRecord(), runnerType: 'Team' as never })).rejects.toThrow( + "Runner state field 'runnerType' must be 'Org' or 'Repo'", + ); + await expect(store.activate('runner-123', { runnerLabels: [''] })).rejects.toThrow( + "Runner state field 'runnerLabels' must be a non-empty string", + ); + await expect(store.recordGitHubIdentity('runner-123', { githubRunnerId: '' })).rejects.toThrow( + "Runner state field 'githubRunnerId' must be a non-empty string", + ); + await expect(store.list({ computeProvider: '' })).rejects.toThrow( + "Runner state field 'computeProvider' must be a non-empty string", + ); + expect(mockDynamoDbClient.calls()).toHaveLength(0); + }); + + it('propagates non-conditional lifecycle errors', async () => { + const error = new Error('service unavailable'); + mockDynamoDbClient.on(UpdateItemCommand).rejects(error); + const store = createAwsDynamoDbRunnerStateStore(); + + await expect(store.beginTermination('runner-123')).rejects.toBe(error); + }); +}); + +function createRecord(): CreateRunnerStateRecord { + return { + runnerId: 'runner-123', + computeProvider: 'aws_ec2', + computeResourceId: 'i-123', + runnerName: 'ghr-runner-123', + runnerLabels: ['linux', 'x64'], + runnerOwner: 'github-aws-runners', + runnerType: 'Org', + metadata: [{ key: 'Environment', value: 'test' }], + }; +} + +function storedRecord(options: { runnerId?: string; resourceId?: string } = {}): Record { + const runnerId = options.runnerId ?? 'runner-123'; + return { + scope: { S: 'entry#linux-x64#runner-state' }, + id: { S: `runner#${runnerId}` }, + runner_id: { S: runnerId }, + compute_provider: { S: 'aws_ec2' }, + compute_resource_id: { S: options.resourceId ?? 'i-123' }, + runner_name: { S: `ghr-${runnerId}` }, + runner_labels: { L: [{ S: 'linux' }, { S: 'x64' }] }, + github_runner_id: { S: '9876' }, + runner_owner: { S: 'github-aws-runners' }, + runner_type: { S: 'Org' }, + state: { S: 'active' }, + created_at: { S: '2025-01-01T00:00:00.000Z' }, + updated_at: { S: '2025-01-01T00:01:00.000Z' }, + metadata: { L: [{ M: { key: { S: 'Environment' }, value: { S: 'test' } } }] }, + }; +} + +function expectedRecord(options: { runnerId?: string; resourceId?: string } = {}) { + const runnerId = options.runnerId ?? 'runner-123'; + return { + runnerId, + computeProvider: 'aws_ec2', + computeResourceId: options.resourceId ?? 'i-123', + runnerName: `ghr-${runnerId}`, + runnerLabels: ['linux', 'x64'], + githubRunnerId: '9876', + runnerOwner: 'github-aws-runners', + runnerType: 'Org', + state: 'active', + createdAt: '2025-01-01T00:00:00.000Z', + updatedAt: '2025-01-01T00:01:00.000Z', + metadata: [{ key: 'Environment', value: 'test' }], + }; +} diff --git a/lambdas/libs/storage-providers/aws/dynamodb/runner-state-store.ts b/lambdas/libs/storage-providers/aws/dynamodb/runner-state-store.ts new file mode 100644 index 0000000000..7263aa93c1 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/dynamodb/runner-state-store.ts @@ -0,0 +1,556 @@ +import { + ConditionalCheckFailedException, + DeleteItemCommand, + PutItemCommand, + QueryCommand, + UpdateItemCommand, + type AttributeValue, +} from '@aws-sdk/client-dynamodb'; + +import type { + CreateRunnerStateRecord, + RunnerConfigMetadata, + RunnerGitHubIdentity, + RunnerLifecycleState, + RunnerStateActivation, + RunnerStateFilter, + RunnerStateRecord, + RunnerStateStore, + RunnerType, +} from '../../core'; +import { getDynamoDbClient } from './client'; +import { positiveIntegerEnvironmentValue, requiredEnvironmentValue } from './environment'; +import { EXPIRES_AT_ATTRIBUTE, ID_ATTRIBUTE, runnerStateId, runnerStateScope, SCOPE_ATTRIBUTE } from './keys'; + +const RUNNER_ID_PREFIX = 'runner#'; +const RUNNER_ID_ATTRIBUTE = 'runner_id'; +const COMPUTE_PROVIDER_ATTRIBUTE = 'compute_provider'; +const COMPUTE_RESOURCE_ID_ATTRIBUTE = 'compute_resource_id'; +const RUNNER_NAME_ATTRIBUTE = 'runner_name'; +const RUNNER_LABELS_ATTRIBUTE = 'runner_labels'; +const GITHUB_RUNNER_ID_ATTRIBUTE = 'github_runner_id'; +const RUNNER_OWNER_ATTRIBUTE = 'runner_owner'; +const RUNNER_TYPE_ATTRIBUTE = 'runner_type'; +const STATE_ATTRIBUTE = 'state'; +const CREATED_AT_ATTRIBUTE = 'created_at'; +const UPDATED_AT_ATTRIBUTE = 'updated_at'; +const METADATA_ATTRIBUTE = 'metadata'; +// AWS Lambda can run for at most 15 minutes. One extra minute prevents a second +// invocation from reclaiming a termination while the original can still be running. +const TERMINATION_CLAIM_LEASE_MILLISECONDS = 16 * 60 * 1000; + +interface AwsDynamoDbRunnerStateStoreConfig { + tableName: string; + scope: string; + ttlSeconds: number; +} + +export function createAwsDynamoDbRunnerStateStore(): RunnerStateStore { + return new AwsDynamoDbRunnerStateStore({ + tableName: requiredEnvironmentValue('RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TABLE_NAME'), + scope: runnerStateScope(requiredEnvironmentValue('RUNNER_CONFIG_DYNAMODB_ENTRY_ID')), + ttlSeconds: positiveIntegerEnvironmentValue('RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TTL_SECONDS'), + }); +} + +class AwsDynamoDbRunnerStateStore implements RunnerStateStore { + constructor(private readonly config: AwsDynamoDbRunnerStateStoreConfig) {} + + async create(record: CreateRunnerStateRecord): Promise { + validateCreateRecord(record); + const now = new Date(); + const timestamp = now.toISOString(); + const item: Record = { + [SCOPE_ATTRIBUTE]: { S: this.config.scope }, + [ID_ATTRIBUTE]: { S: runnerStateId(record.runnerId) }, + [RUNNER_ID_ATTRIBUTE]: { S: record.runnerId }, + [COMPUTE_PROVIDER_ATTRIBUTE]: { S: record.computeProvider }, + [COMPUTE_RESOURCE_ID_ATTRIBUTE]: { S: record.computeResourceId }, + [RUNNER_OWNER_ATTRIBUTE]: { S: record.runnerOwner }, + [RUNNER_TYPE_ATTRIBUTE]: { S: record.runnerType }, + [STATE_ATTRIBUTE]: { S: 'provisioning' }, + [CREATED_AT_ATTRIBUTE]: { S: timestamp }, + [UPDATED_AT_ATTRIBUTE]: { S: timestamp }, + [EXPIRES_AT_ATTRIBUTE]: { N: expiresAt(now, this.config.ttlSeconds) }, + }; + + setOptionalString(item, RUNNER_NAME_ATTRIBUTE, record.runnerName); + setOptionalStringList(item, RUNNER_LABELS_ATTRIBUTE, record.runnerLabels); + setMetadata(item, record.metadata); + + await getDynamoDbClient().send( + new PutItemCommand({ + TableName: this.config.tableName, + Item: item, + ConditionExpression: 'attribute_not_exists(#scope) AND attribute_not_exists(#id)', + ExpressionAttributeNames: { + '#scope': SCOPE_ATTRIBUTE, + '#id': ID_ATTRIBUTE, + }, + }), + ); + } + + async activate(runnerId: string, activation: RunnerStateActivation = {}): Promise { + validateNonEmptyString(runnerId, 'runnerId'); + validateActivation(activation); + await this.transition(runnerId, ['provisioning'], 'active', activation); + } + + async recordGitHubIdentity(runnerId: string, identity: RunnerGitHubIdentity): Promise { + validateNonEmptyString(identity.githubRunnerId, 'githubRunnerId'); + validateActivation(identity); + await this.transition(runnerId, ['provisioning'], 'provisioning', identity); + } + + async list(filter: RunnerStateFilter = {}): Promise { + if (filter.computeProvider !== undefined) { + validateNonEmptyString(filter.computeProvider, 'computeProvider'); + } + + const records: RunnerStateRecord[] = []; + let exclusiveStartKey: Record | undefined; + + do { + const expressionAttributeNames: Record = { + '#scope': SCOPE_ATTRIBUTE, + '#id': ID_ATTRIBUTE, + }; + const expressionAttributeValues: Record = { + ':scope': { S: this.config.scope }, + ':runner': { S: RUNNER_ID_PREFIX }, + }; + + if (filter.computeProvider !== undefined) { + expressionAttributeNames['#computeProvider'] = COMPUTE_PROVIDER_ATTRIBUTE; + expressionAttributeValues[':computeProvider'] = { S: filter.computeProvider }; + } + + const result = await getDynamoDbClient().send( + new QueryCommand({ + TableName: this.config.tableName, + KeyConditionExpression: '#scope = :scope AND begins_with(#id, :runner)', + FilterExpression: filter.computeProvider === undefined ? undefined : '#computeProvider = :computeProvider', + ExpressionAttributeNames: expressionAttributeNames, + ExpressionAttributeValues: expressionAttributeValues, + ConsistentRead: true, + ExclusiveStartKey: exclusiveStartKey, + }), + ); + + for (const item of result.Items ?? []) { + records.push(parseRunnerStateRecord(item, this.config.scope)); + } + exclusiveStartKey = result.LastEvaluatedKey; + } while (exclusiveStartKey !== undefined); + + return records; + } + + async markOrphan(runnerId: string): Promise { + await this.transition(runnerId, ['active'], 'orphan'); + } + + async unmarkOrphan(runnerId: string): Promise { + await this.transition(runnerId, ['orphan'], 'active'); + } + + async beginTermination(runnerId: string): Promise { + validateNonEmptyString(runnerId, 'runnerId'); + const now = new Date(); + const staleBefore = new Date(now.getTime() - TERMINATION_CLAIM_LEASE_MILLISECONDS).toISOString(); + try { + const previous = ( + await getDynamoDbClient().send( + new UpdateItemCommand({ + TableName: this.config.tableName, + Key: this.key(runnerId), + UpdateExpression: 'SET #state = :terminating, #updatedAt = :updatedAt, #expiresAt = :expiresAt', + ConditionExpression: + 'attribute_exists(#scope) AND attribute_exists(#id) AND (#state IN (:provisioning, :active, :orphan) OR (#state = :terminating AND #updatedAt < :staleBefore))', + ExpressionAttributeNames: { + '#scope': SCOPE_ATTRIBUTE, + '#id': ID_ATTRIBUTE, + '#state': STATE_ATTRIBUTE, + '#updatedAt': UPDATED_AT_ATTRIBUTE, + '#expiresAt': EXPIRES_AT_ATTRIBUTE, + }, + ExpressionAttributeValues: { + ':provisioning': { S: 'provisioning' }, + ':active': { S: 'active' }, + ':orphan': { S: 'orphan' }, + ':terminating': { S: 'terminating' }, + ':updatedAt': { S: now.toISOString() }, + ':staleBefore': { S: staleBefore }, + ':expiresAt': { N: expiresAt(now, this.config.ttlSeconds) }, + }, + ReturnValues: 'ALL_OLD', + }), + ) + ).Attributes; + const previousState = previous?.[STATE_ATTRIBUTE]?.S; + if (previousState === undefined || !isRunnerLifecycleState(previousState)) { + throw new Error(`Runner state item '${this.config.scope}/${runnerStateId(runnerId)}' returned no prior state`); + } + return previousState; + } catch (error) { + if (error instanceof ConditionalCheckFailedException) { + return undefined; + } + throw error; + } + } + + async cancelTermination(runnerId: string, restoreState: 'provisioning' | 'active' | 'orphan'): Promise { + if (restoreState !== 'provisioning' && restoreState !== 'active' && restoreState !== 'orphan') { + throw new Error("Runner state field 'restoreState' must be 'provisioning', 'active', or 'orphan'"); + } + await this.transition(runnerId, ['terminating'], restoreState); + } + + async delete(runnerId: string): Promise { + validateNonEmptyString(runnerId, 'runnerId'); + await getDynamoDbClient().send( + new DeleteItemCommand({ + TableName: this.config.tableName, + Key: this.key(runnerId), + ConditionExpression: '#state = :terminating', + ExpressionAttributeNames: { + '#state': STATE_ATTRIBUTE, + }, + ExpressionAttributeValues: { + ':terminating': { S: 'terminating' }, + }, + }), + ); + } + + private async transition( + runnerId: string, + expectedStates: RunnerLifecycleState[], + state: RunnerLifecycleState, + activation: RunnerStateActivation = {}, + returnOldState = false, + ): Promise | undefined> { + validateNonEmptyString(runnerId, 'runnerId'); + const now = new Date(); + const expressionAttributeNames: Record = { + '#scope': SCOPE_ATTRIBUTE, + '#id': ID_ATTRIBUTE, + '#state': STATE_ATTRIBUTE, + '#updatedAt': UPDATED_AT_ATTRIBUTE, + '#expiresAt': EXPIRES_AT_ATTRIBUTE, + }; + const expressionAttributeValues: Record = { + ':state': { S: state }, + ':updatedAt': { S: now.toISOString() }, + }; + const updates = ['#state = :state', '#updatedAt = :updatedAt']; + const hasSafetyTtl = state === 'provisioning' || state === 'terminating'; + if (hasSafetyTtl) { + expressionAttributeValues[':expiresAt'] = { N: expiresAt(now, this.config.ttlSeconds) }; + updates.push('#expiresAt = :expiresAt'); + } + addActivationUpdates(updates, expressionAttributeNames, expressionAttributeValues, activation); + + const expectedStateValues = expectedStates.map((expectedState, index) => { + const placeholder = `:expectedState${index}`; + expressionAttributeValues[placeholder] = { S: expectedState }; + return placeholder; + }); + + const result = await getDynamoDbClient().send( + new UpdateItemCommand({ + TableName: this.config.tableName, + Key: this.key(runnerId), + UpdateExpression: `SET ${updates.join(', ')}${hasSafetyTtl ? '' : ' REMOVE #expiresAt'}`, + ConditionExpression: `attribute_exists(#scope) AND attribute_exists(#id) AND #state IN (${expectedStateValues.join(', ')})`, + ExpressionAttributeNames: expressionAttributeNames, + ExpressionAttributeValues: expressionAttributeValues, + ReturnValues: returnOldState ? 'ALL_OLD' : undefined, + }), + ); + return result.Attributes; + } + + private key(runnerId: string): Record { + return { + [SCOPE_ATTRIBUTE]: { S: this.config.scope }, + [ID_ATTRIBUTE]: { S: runnerStateId(runnerId) }, + }; + } +} + +function parseRunnerStateRecord(item: Record, scope: string): RunnerStateRecord { + const id = requiredStringAttribute(item, ID_ATTRIBUTE, scope); + const runnerType = requiredStringAttribute(item, RUNNER_TYPE_ATTRIBUTE, `${scope}/${id}`); + if (runnerType !== 'Org' && runnerType !== 'Repo') { + throw invalidItem(`${scope}/${id}`, RUNNER_TYPE_ATTRIBUTE); + } + + return { + runnerId: requiredStringAttribute(item, RUNNER_ID_ATTRIBUTE, `${scope}/${id}`), + computeProvider: requiredStringAttribute(item, COMPUTE_PROVIDER_ATTRIBUTE, `${scope}/${id}`), + computeResourceId: requiredStringAttribute(item, COMPUTE_RESOURCE_ID_ATTRIBUTE, `${scope}/${id}`), + runnerName: optionalStringAttribute(item, RUNNER_NAME_ATTRIBUTE, `${scope}/${id}`), + runnerLabels: optionalStringListAttribute(item, RUNNER_LABELS_ATTRIBUTE, `${scope}/${id}`), + githubRunnerId: optionalStringAttribute(item, GITHUB_RUNNER_ID_ATTRIBUTE, `${scope}/${id}`), + runnerOwner: requiredStringAttribute(item, RUNNER_OWNER_ATTRIBUTE, `${scope}/${id}`), + runnerType, + state: requiredLifecycleState(item, `${scope}/${id}`), + createdAt: requiredTimestampAttribute(item, CREATED_AT_ATTRIBUTE, `${scope}/${id}`), + updatedAt: requiredTimestampAttribute(item, UPDATED_AT_ATTRIBUTE, `${scope}/${id}`), + metadata: optionalMetadataAttribute(item, `${scope}/${id}`), + }; +} + +function requiredLifecycleState(item: Record, itemId: string): RunnerLifecycleState { + const state = requiredStringAttribute(item, STATE_ATTRIBUTE, itemId); + if (!isRunnerLifecycleState(state)) { + throw invalidItem(itemId, STATE_ATTRIBUTE); + } + return state; +} + +function isRunnerLifecycleState(value: string): value is RunnerLifecycleState { + return value === 'provisioning' || value === 'active' || value === 'orphan' || value === 'terminating'; +} + +function requiredStringAttribute(item: Record, name: string, itemId: string): string { + const value = item[name]?.S; + if (value === undefined || value.trim() === '') { + throw invalidItem(itemId, name); + } + return value; +} + +function optionalStringAttribute( + item: Record, + name: string, + itemId: string, +): string | undefined { + if (item[name] === undefined) { + return undefined; + } + return requiredStringAttribute(item, name, itemId); +} + +function optionalStringListAttribute( + item: Record, + name: string, + itemId: string, +): string[] | undefined { + const attribute = item[name]; + if (attribute === undefined) { + return undefined; + } + if (!attribute.L) { + throw invalidItem(itemId, name); + } + + return attribute.L.map((value) => { + if (value.S === undefined || value.S.trim() === '') { + throw invalidItem(itemId, name); + } + return value.S; + }); +} + +function requiredTimestampAttribute(item: Record, name: string, itemId: string): string { + const value = requiredStringAttribute(item, name, itemId); + try { + if (new Date(value).toISOString() !== value) { + throw invalidItem(itemId, name); + } + } catch { + throw invalidItem(itemId, name); + } + return value; +} + +function optionalMetadataAttribute( + item: Record, + itemId: string, +): RunnerConfigMetadata[] | undefined { + const metadata = item[METADATA_ATTRIBUTE]; + if (metadata === undefined) { + return undefined; + } + if (!metadata.L) { + throw invalidItem(itemId, METADATA_ATTRIBUTE); + } + + return metadata.L.map((entry) => { + if (!entry.M) { + throw invalidItem(itemId, METADATA_ATTRIBUTE); + } + return { + key: requiredStringAttribute(entry.M, 'key', itemId), + value: requiredMetadataValue(entry.M, itemId), + }; + }); +} + +function validateCreateRecord(record: CreateRunnerStateRecord): void { + validateNonEmptyString(record.runnerId, 'runnerId'); + validateNonEmptyString(record.computeProvider, 'computeProvider'); + validateNonEmptyString(record.computeResourceId, 'computeResourceId'); + validateOptionalString(record.runnerName, 'runnerName'); + validateOptionalStringList(record.runnerLabels, 'runnerLabels'); + validateNonEmptyString(record.runnerOwner, 'runnerOwner'); + validateRunnerType(record.runnerType); + for (const metadata of record.metadata ?? []) { + validateNonEmptyString(metadata.key, 'metadata.key'); + validateString(metadata.value, 'metadata.value'); + } +} + +function validateActivation(activation: RunnerStateActivation): void { + validateOptionalString(activation.runnerName, 'runnerName'); + validateOptionalStringList(activation.runnerLabels, 'runnerLabels'); + validateOptionalString(activation.githubRunnerId, 'githubRunnerId'); + for (const metadata of activation.metadata ?? []) { + validateNonEmptyString(metadata.key, 'metadata.key'); + validateString(metadata.value, 'metadata.value'); + } +} + +function validateRunnerType(value: RunnerType): void { + if (value !== 'Org' && value !== 'Repo') { + throw new Error("Runner state field 'runnerType' must be 'Org' or 'Repo'"); + } +} + +function validateOptionalString(value: string | undefined, name: string): void { + if (value !== undefined) { + validateNonEmptyString(value, name); + } +} + +function validateOptionalStringList(values: string[] | undefined, name: string): void { + for (const value of values ?? []) { + validateNonEmptyString(value, name); + } +} + +function validateNonEmptyString(value: string, name: string): void { + if (typeof value !== 'string' || value.trim() === '') { + throw new Error(`Runner state field '${name}' must be a non-empty string`); + } +} + +function validateString(value: string, name: string): void { + if (typeof value !== 'string') { + throw new Error(`Runner state field '${name}' must be a string`); + } +} + +function requiredMetadataValue(item: Record, itemId: string): string { + const value = item.value?.S; + if (value === undefined) { + throw invalidItem(itemId, METADATA_ATTRIBUTE); + } + return value; +} + +function setOptionalString(item: Record, name: string, value: string | undefined): void { + if (value !== undefined) { + item[name] = { S: value }; + } +} + +function setOptionalStringList(item: Record, name: string, values: string[] | undefined): void { + if (values !== undefined) { + item[name] = { L: values.map((value) => ({ S: value })) }; + } +} + +function addActivationUpdates( + updates: string[], + names: Record, + values: Record, + activation: RunnerStateActivation, +): void { + addOptionalStringUpdate( + updates, + names, + values, + '#runnerName', + ':runnerName', + RUNNER_NAME_ATTRIBUTE, + activation.runnerName, + ); + addOptionalStringListUpdate( + updates, + names, + values, + '#runnerLabels', + ':runnerLabels', + RUNNER_LABELS_ATTRIBUTE, + activation.runnerLabels, + ); + addOptionalStringUpdate( + updates, + names, + values, + '#githubRunnerId', + ':githubRunnerId', + GITHUB_RUNNER_ID_ATTRIBUTE, + activation.githubRunnerId, + ); + if (activation.metadata !== undefined) { + names['#metadata'] = METADATA_ATTRIBUTE; + values[':metadata'] = { + L: activation.metadata.map(({ key, value }) => ({ M: { key: { S: key }, value: { S: value } } })), + }; + updates.push('#metadata = :metadata'); + } +} + +function addOptionalStringUpdate( + updates: string[], + names: Record, + values: Record, + namePlaceholder: string, + valuePlaceholder: string, + attributeName: string, + value: string | undefined, +): void { + if (value !== undefined) { + names[namePlaceholder] = attributeName; + values[valuePlaceholder] = { S: value }; + updates.push(`${namePlaceholder} = ${valuePlaceholder}`); + } +} + +function addOptionalStringListUpdate( + updates: string[], + names: Record, + values: Record, + namePlaceholder: string, + valuePlaceholder: string, + attributeName: string, + value: string[] | undefined, +): void { + if (value !== undefined) { + names[namePlaceholder] = attributeName; + values[valuePlaceholder] = { L: value.map((entry) => ({ S: entry })) }; + updates.push(`${namePlaceholder} = ${valuePlaceholder}`); + } +} + +function setMetadata(item: Record, metadata: RunnerConfigMetadata[] | undefined): void { + if (metadata && metadata.length > 0) { + item[METADATA_ATTRIBUTE] = { + L: metadata.map(({ key, value }) => ({ M: { key: { S: key }, value: { S: value } } })), + }; + } +} + +function expiresAt(now: Date, ttlSeconds: number): string { + return (Math.floor(now.getTime() / 1000) + ttlSeconds).toString(); +} + +function invalidItem(itemId: string, attribute: string): Error { + return new Error(`Runner state item '${itemId}' has an invalid '${attribute}' attribute`); +} diff --git a/lambdas/libs/storage-providers/core/index.ts b/lambdas/libs/storage-providers/core/index.ts index e6060eae2a..cf005b478c 100644 --- a/lambdas/libs/storage-providers/core/index.ts +++ b/lambdas/libs/storage-providers/core/index.ts @@ -9,6 +9,7 @@ export interface RunnerConfigMetadata { export interface RunnerConfigRecord { runnerId: string; value: string; + accessScope?: string; } export interface RunnerConfigStore { @@ -65,3 +66,53 @@ export interface RunnerGroupCacheStore { export interface RunnerMatcherConfigStore { get(): Promise; } + +export type RunnerType = 'Org' | 'Repo'; +export type RunnerLifecycleState = 'provisioning' | 'active' | 'orphan' | 'terminating'; + +export interface RunnerStateRecord { + runnerId: string; + computeProvider: string; + computeResourceId: string; + runnerName?: string; + runnerLabels?: string[]; + githubRunnerId?: string; + runnerOwner: string; + runnerType: RunnerType; + state: RunnerLifecycleState; + createdAt: string; + updatedAt: string; + metadata?: RunnerConfigMetadata[]; +} + +export type CreateRunnerStateRecord = Omit; + +export interface RunnerStateActivation { + runnerName?: string; + runnerLabels?: string[]; + githubRunnerId?: string; + metadata?: RunnerConfigMetadata[]; +} + +export type RunnerGitHubIdentity = RunnerStateActivation & { githubRunnerId: string }; + +export interface RunnerStateFilter { + computeProvider?: string; +} + +/** + * Provider-neutral index of compute resources that implement GitHub runners. + * Runner bootstrap configuration is deliberately stored by RunnerConfigStore + * in a separate item because it contains a short-lived secret payload. + */ +export interface RunnerStateStore { + create(record: CreateRunnerStateRecord): Promise; + recordGitHubIdentity(runnerId: string, identity: RunnerGitHubIdentity): Promise; + activate(runnerId: string, activation?: RunnerStateActivation): Promise; + list(filter?: RunnerStateFilter): Promise; + markOrphan(runnerId: string): Promise; + unmarkOrphan(runnerId: string): Promise; + beginTermination(runnerId: string): Promise; + cancelTermination(runnerId: string, restoreState: 'provisioning' | 'active' | 'orphan'): Promise; + delete(runnerId: string): Promise; +} diff --git a/lambdas/libs/storage-providers/github-app-credentials.test.ts b/lambdas/libs/storage-providers/github-app-credentials.test.ts new file mode 100644 index 0000000000..8b28f4ae8e --- /dev/null +++ b/lambdas/libs/storage-providers/github-app-credentials.test.ts @@ -0,0 +1,97 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAwsDynamoDbGitHubAppCredentialsStore } from './aws/dynamodb/github-app-credentials-store'; +import { createAwsSsmGitHubAppCredentialsStore } from './aws/ssm/github-app-credentials-store'; +import type { GitHubAppCredentialsStore } from './core'; +import { getGitHubAppCredentialsStore, resetGitHubAppCredentialsStore } from './github-app-credentials'; + +vi.mock('./aws/dynamodb/github-app-credentials-store', () => ({ + createAwsDynamoDbGitHubAppCredentialsStore: vi.fn(), +})); +vi.mock('./aws/ssm/github-app-credentials-store', () => ({ + createAwsSsmGitHubAppCredentialsStore: vi.fn(), +})); + +const createAwsDynamoDbStoreMock = vi.mocked(createAwsDynamoDbGitHubAppCredentialsStore); +const createAwsSsmStoreMock = vi.mocked(createAwsSsmGitHubAppCredentialsStore); +const cleanEnv = process.env; + +describe('GitHub App credentials store selection', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env = { ...cleanEnv }; + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + resetGitHubAppCredentialsStore(); + }); + + it.each([undefined, '', ' ', 'aws_ssm', ' AWS_SSM '])('uses aws_ssm for selector input %j', (provider) => { + setProvider(provider); + const store = stubSsmStore(); + + expect(getGitHubAppCredentialsStore()).toBe(store); + expect(createAwsSsmStoreMock).toHaveBeenCalledOnce(); + expect(createAwsDynamoDbStoreMock).not.toHaveBeenCalled(); + }); + + it.each(['aws_dynamodb', ' AWS_DYNAMODB '])('uses aws_dynamodb for selector input %j', (provider) => { + setProvider(provider); + const store = stubDynamoDbStore(); + + expect(getGitHubAppCredentialsStore()).toBe(store); + expect(createAwsDynamoDbStoreMock).toHaveBeenCalledOnce(); + expect(createAwsSsmStoreMock).not.toHaveBeenCalled(); + }); + + it('rejects an unsupported provider before creating a store', () => { + setProvider('not-registered'); + + expect(() => getGitHubAppCredentialsStore()).toThrow("Unsupported runner config storage provider 'not-registered'"); + expect(createAwsSsmStoreMock).not.toHaveBeenCalled(); + expect(createAwsDynamoDbStoreMock).not.toHaveBeenCalled(); + }); + + it('creates the selected store lazily and caches it', () => { + const store = stubSsmStore(); + + expect(createAwsSsmStoreMock).not.toHaveBeenCalled(); + const first = getGitHubAppCredentialsStore(); + setProvider('not-registered'); + const second = getGitHubAppCredentialsStore(); + + expect(first).toBe(store); + expect(second).toBe(store); + expect(createAwsSsmStoreMock).toHaveBeenCalledOnce(); + }); + + it('selects again after the test reset', () => { + const firstStore = stubSsmStore(); + expect(getGitHubAppCredentialsStore()).toBe(firstStore); + + const secondStore = { get: vi.fn() } satisfies GitHubAppCredentialsStore; + createAwsSsmStoreMock.mockReturnValue(secondStore); + resetGitHubAppCredentialsStore(); + + expect(getGitHubAppCredentialsStore()).toBe(secondStore); + expect(createAwsSsmStoreMock).toHaveBeenCalledTimes(2); + }); +}); + +function setProvider(provider: string | undefined): void { + if (provider === undefined) { + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + } else { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + } +} + +function stubSsmStore(): GitHubAppCredentialsStore { + const store = { get: vi.fn() } satisfies GitHubAppCredentialsStore; + createAwsSsmStoreMock.mockReturnValue(store); + return store; +} + +function stubDynamoDbStore(): GitHubAppCredentialsStore { + const store = { get: vi.fn() } satisfies GitHubAppCredentialsStore; + createAwsDynamoDbStoreMock.mockReturnValue(store); + return store; +} diff --git a/lambdas/libs/storage-providers/github-app-credentials.ts b/lambdas/libs/storage-providers/github-app-credentials.ts new file mode 100644 index 0000000000..11322d826d --- /dev/null +++ b/lambdas/libs/storage-providers/github-app-credentials.ts @@ -0,0 +1,25 @@ +import { createAwsDynamoDbGitHubAppCredentialsStore } from './aws/dynamodb/github-app-credentials-store'; +import { createAwsSsmGitHubAppCredentialsStore } from './aws/ssm/github-app-credentials-store'; +import type { GitHubAppCredentialsStore } from './core'; +import type {} from './environment'; +import { resolveRunnerConfigStorageProvider, type RunnerConfigStorageProvider } from './provider'; + +type GitHubAppCredentialsStoreFactory = () => GitHubAppCredentialsStore; + +const providerFactories = { + aws_ssm: createAwsSsmGitHubAppCredentialsStore, + aws_dynamodb: createAwsDynamoDbGitHubAppCredentialsStore, +} as const satisfies Record; + +let githubAppCredentialsStore: GitHubAppCredentialsStore | undefined; + +export function getGitHubAppCredentialsStore(): GitHubAppCredentialsStore { + githubAppCredentialsStore ??= + providerFactories[resolveRunnerConfigStorageProvider(process.env.RUNNER_CONFIG_STORAGE_PROVIDER)](); + return githubAppCredentialsStore; +} + +// Test-only reset for cases that need to exercise first-use environment selection. +export function resetGitHubAppCredentialsStore(): void { + githubAppCredentialsStore = undefined; +} diff --git a/lambdas/libs/storage-providers/github-webhook-secret.test.ts b/lambdas/libs/storage-providers/github-webhook-secret.test.ts index 5888e735b0..9b66f840be 100644 --- a/lambdas/libs/storage-providers/github-webhook-secret.test.ts +++ b/lambdas/libs/storage-providers/github-webhook-secret.test.ts @@ -1,14 +1,19 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { createAwsDynamoDbGitHubWebhookSecretStore } from './aws/dynamodb/github-webhook-secret-store'; import { createAwsSsmGitHubWebhookSecretStore } from './aws/ssm/github-webhook-secret-store'; import type { GitHubWebhookSecretStore } from './core'; import { getGitHubWebhookSecretStore, resetGitHubWebhookSecretStore } from './github-webhook-secret'; +vi.mock('./aws/dynamodb/github-webhook-secret-store', () => ({ + createAwsDynamoDbGitHubWebhookSecretStore: vi.fn(), +})); vi.mock('./aws/ssm/github-webhook-secret-store', () => ({ createAwsSsmGitHubWebhookSecretStore: vi.fn(), })); -const createAwsSsmGitHubWebhookSecretStoreMock = vi.mocked(createAwsSsmGitHubWebhookSecretStore); +const createAwsDynamoDbStoreMock = vi.mocked(createAwsDynamoDbGitHubWebhookSecretStore); +const createAwsSsmStoreMock = vi.mocked(createAwsSsmGitHubWebhookSecretStore); const cleanEnv = process.env; describe('GitHub webhook secret store selection', () => { @@ -19,52 +24,55 @@ describe('GitHub webhook secret store selection', () => { resetGitHubWebhookSecretStore(); }); - it.each([undefined, '', ' '])('uses aws_ssm for default selector input %j', (provider) => { + it.each([undefined, '', ' ', 'aws_ssm', ' AWS_SSM '])('uses aws_ssm for selector input %j', (provider) => { setProvider(provider); - const store = stubStore(); + const store = stubSsmStore(); expect(getGitHubWebhookSecretStore()).toBe(store); - expect(createAwsSsmGitHubWebhookSecretStoreMock).toHaveBeenCalledOnce(); + expect(createAwsSsmStoreMock).toHaveBeenCalledOnce(); + expect(createAwsDynamoDbStoreMock).not.toHaveBeenCalled(); }); - it.each(['aws_ssm', ' AWS_SSM '])('uses aws_ssm for explicit selector input %j', (provider) => { - process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; - const store = stubStore(); + it.each(['aws_dynamodb', ' AWS_DYNAMODB '])('uses aws_dynamodb for selector input %j', (provider) => { + setProvider(provider); + const store = stubDynamoDbStore(); expect(getGitHubWebhookSecretStore()).toBe(store); - expect(createAwsSsmGitHubWebhookSecretStoreMock).toHaveBeenCalledOnce(); + expect(createAwsDynamoDbStoreMock).toHaveBeenCalledOnce(); + expect(createAwsSsmStoreMock).not.toHaveBeenCalled(); }); - it('rejects an unsupported provider on first use', () => { - process.env.RUNNER_CONFIG_STORAGE_PROVIDER = 'not-registered'; + it('rejects an unsupported provider before creating a store', () => { + setProvider('not-registered'); expect(() => getGitHubWebhookSecretStore()).toThrow("Unsupported runner config storage provider 'not-registered'"); - expect(createAwsSsmGitHubWebhookSecretStoreMock).not.toHaveBeenCalled(); + expect(createAwsSsmStoreMock).not.toHaveBeenCalled(); + expect(createAwsDynamoDbStoreMock).not.toHaveBeenCalled(); }); - it('selects lazily and caches the created store', () => { - const store = stubStore(); + it('creates the selected store lazily and caches it', () => { + const store = stubSsmStore(); - expect(createAwsSsmGitHubWebhookSecretStoreMock).not.toHaveBeenCalled(); + expect(createAwsSsmStoreMock).not.toHaveBeenCalled(); const first = getGitHubWebhookSecretStore(); - process.env.RUNNER_CONFIG_STORAGE_PROVIDER = 'not-registered'; + setProvider('not-registered'); const second = getGitHubWebhookSecretStore(); expect(first).toBe(store); expect(second).toBe(store); - expect(createAwsSsmGitHubWebhookSecretStoreMock).toHaveBeenCalledOnce(); + expect(createAwsSsmStoreMock).toHaveBeenCalledOnce(); }); it('selects again after the test reset', () => { - const firstStore = stubStore(); + const firstStore = stubSsmStore(); expect(getGitHubWebhookSecretStore()).toBe(firstStore); const secondStore = { get: vi.fn() } satisfies GitHubWebhookSecretStore; - createAwsSsmGitHubWebhookSecretStoreMock.mockReturnValue(secondStore); + createAwsSsmStoreMock.mockReturnValue(secondStore); resetGitHubWebhookSecretStore(); expect(getGitHubWebhookSecretStore()).toBe(secondStore); - expect(createAwsSsmGitHubWebhookSecretStoreMock).toHaveBeenCalledTimes(2); + expect(createAwsSsmStoreMock).toHaveBeenCalledTimes(2); }); }); @@ -76,8 +84,14 @@ function setProvider(provider: string | undefined): void { } } -function stubStore(): GitHubWebhookSecretStore { +function stubSsmStore(): GitHubWebhookSecretStore { + const store = { get: vi.fn() } satisfies GitHubWebhookSecretStore; + createAwsSsmStoreMock.mockReturnValue(store); + return store; +} + +function stubDynamoDbStore(): GitHubWebhookSecretStore { const store = { get: vi.fn() } satisfies GitHubWebhookSecretStore; - createAwsSsmGitHubWebhookSecretStoreMock.mockReturnValue(store); + createAwsDynamoDbStoreMock.mockReturnValue(store); return store; } diff --git a/lambdas/libs/storage-providers/github-webhook-secret.ts b/lambdas/libs/storage-providers/github-webhook-secret.ts index f13df08718..8fd0a1758c 100644 --- a/lambdas/libs/storage-providers/github-webhook-secret.ts +++ b/lambdas/libs/storage-providers/github-webhook-secret.ts @@ -1,3 +1,4 @@ +import { createAwsDynamoDbGitHubWebhookSecretStore } from './aws/dynamodb/github-webhook-secret-store'; import { createAwsSsmGitHubWebhookSecretStore } from './aws/ssm/github-webhook-secret-store'; import type { GitHubWebhookSecretStore } from './core'; import type {} from './environment'; @@ -7,6 +8,7 @@ type GitHubWebhookSecretStoreFactory = () => GitHubWebhookSecretStore; const providerFactories = { aws_ssm: createAwsSsmGitHubWebhookSecretStore, + aws_dynamodb: createAwsDynamoDbGitHubWebhookSecretStore, } as const satisfies Record; let githubWebhookSecretStore: GitHubWebhookSecretStore | undefined; diff --git a/lambdas/libs/storage-providers/index.ts b/lambdas/libs/storage-providers/index.ts index fcd4bd9338..01d4c3ecf0 100644 --- a/lambdas/libs/storage-providers/index.ts +++ b/lambdas/libs/storage-providers/index.ts @@ -1,4 +1,5 @@ export type { + CreateRunnerStateRecord, GitHubAppCredential, GitHubAppCredentialsStore, GitHubWebhookSecretStore, @@ -10,7 +11,14 @@ export type { RunnerConfigStore, RunnerGroupCacheRecord, RunnerGroupCacheStore, + RunnerGitHubIdentity, RunnerMatcherConfigStore, + RunnerLifecycleState, + RunnerStateActivation, + RunnerStateFilter, + RunnerStateRecord, + RunnerStateStore, + RunnerType, } from './core'; export { createRunnerConfigHousekeeper } from './runner-config-housekeeper'; export { createRunnerConfigConsumer, type RunnerConfigConsumerConfig } from './runner-config-consumer'; @@ -22,5 +30,9 @@ export { export type { RunnerConfigStorageProvider } from './provider'; export { createCommonStorage, createStorageProviders } from './storage-providers'; export type { StorageProviders, RunnerConfigStorage, CommonStorage } from './core'; +export { getGitHubAppCredentialsStore, resetGitHubAppCredentialsStore } from './github-app-credentials'; +export { getRunnerConfigStore, resetRunnerConfigStore } from './runner-config'; +export { getRunnerGroupCacheStore, resetRunnerGroupCacheStore } from './runner-group-cache'; export { getGitHubWebhookSecretStore, resetGitHubWebhookSecretStore } from './github-webhook-secret'; export { getRunnerMatcherConfigStore, resetRunnerMatcherConfigStore } from './runner-matcher-config'; +export { getRunnerStateStore, resetRunnerStateStore } from './runner-state'; diff --git a/lambdas/libs/storage-providers/package.json b/lambdas/libs/storage-providers/package.json index 330d60a3f6..1a8687ef32 100644 --- a/lambdas/libs/storage-providers/package.json +++ b/lambdas/libs/storage-providers/package.json @@ -20,6 +20,7 @@ "dependencies": { "@aws-github-runner/aws-powertools-util": "*", "@aws-github-runner/aws-ssm-util": "*", + "@aws-sdk/client-dynamodb": "^3.1009.0", "@aws-sdk/client-ssm": "^3.1009.0" }, "devDependencies": { diff --git a/lambdas/libs/storage-providers/provider.ts b/lambdas/libs/storage-providers/provider.ts index ff9ecb73cc..9a36fb8f69 100644 --- a/lambdas/libs/storage-providers/provider.ts +++ b/lambdas/libs/storage-providers/provider.ts @@ -1,7 +1,11 @@ export const runnerConfigStorageProvider = { awsSsm: 'aws_ssm', + awsDynamodb: 'aws_dynamodb', } as const; -export const runnerConfigStorageProviders = [runnerConfigStorageProvider.awsSsm] as const; +export const runnerConfigStorageProviders = [ + runnerConfigStorageProvider.awsSsm, + runnerConfigStorageProvider.awsDynamodb, +] as const; export type RunnerConfigStorageProvider = (typeof runnerConfigStorageProviders)[number]; diff --git a/lambdas/libs/storage-providers/runner-config.test.ts b/lambdas/libs/storage-providers/runner-config.test.ts new file mode 100644 index 0000000000..0c85093fa0 --- /dev/null +++ b/lambdas/libs/storage-providers/runner-config.test.ts @@ -0,0 +1,108 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAwsDynamoDbRunnerConfigStore } from './aws/dynamodb/runner-config-store'; +import { createAwsSsmRunnerConfigStore } from './aws/ssm/runner-config-store'; +import type { RunnerConfigStore } from './core'; +import { getRunnerConfigStore, resetRunnerConfigStore } from './runner-config'; + +vi.mock('./aws/dynamodb/runner-config-store', () => ({ + createAwsDynamoDbRunnerConfigStore: vi.fn(), +})); +vi.mock('./aws/ssm/runner-config-store', () => ({ + createAwsSsmRunnerConfigStore: vi.fn(), +})); + +const createAwsDynamoDbRunnerConfigStoreMock = vi.mocked(createAwsDynamoDbRunnerConfigStore); +const createAwsSsmRunnerConfigStoreMock = vi.mocked(createAwsSsmRunnerConfigStore); +const cleanEnv = process.env; + +describe('runner config store selection', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env = { ...cleanEnv }; + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + resetRunnerConfigStore(); + }); + + it.each([undefined, '', ' '])('uses aws_ssm for default selector input %j', (provider) => { + setProvider(provider); + const store = stubStore(); + + expect(getRunnerConfigStore()).toBe(store); + expect(createAwsSsmRunnerConfigStoreMock).toHaveBeenCalledOnce(); + expect(createAwsDynamoDbRunnerConfigStoreMock).not.toHaveBeenCalled(); + }); + + it.each(['aws_ssm', ' AWS_SSM '])('uses aws_ssm for explicit selector input %j', (provider) => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + const store = stubStore(); + + expect(getRunnerConfigStore()).toBe(store); + expect(createAwsSsmRunnerConfigStoreMock).toHaveBeenCalledOnce(); + expect(createAwsDynamoDbRunnerConfigStoreMock).not.toHaveBeenCalled(); + }); + + it.each(['aws_dynamodb', ' AWS_DYNAMODB '])('uses aws_dynamodb for explicit selector input %j', (provider) => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + const store = stubDynamoDbStore(); + + expect(getRunnerConfigStore()).toBe(store); + expect(createAwsDynamoDbRunnerConfigStoreMock).toHaveBeenCalledOnce(); + expect(createAwsSsmRunnerConfigStoreMock).not.toHaveBeenCalled(); + }); + + it('rejects an unsupported provider on first use', () => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = 'not-registered'; + + expect(createAwsSsmRunnerConfigStoreMock).not.toHaveBeenCalled(); + expect(createAwsDynamoDbRunnerConfigStoreMock).not.toHaveBeenCalled(); + expect(() => getRunnerConfigStore()).toThrow("Unsupported runner config storage provider 'not-registered'"); + expect(createAwsSsmRunnerConfigStoreMock).not.toHaveBeenCalled(); + expect(createAwsDynamoDbRunnerConfigStoreMock).not.toHaveBeenCalled(); + }); + + it('selects lazily and caches the created store', () => { + const store = stubStore(); + + expect(createAwsSsmRunnerConfigStoreMock).not.toHaveBeenCalled(); + const first = getRunnerConfigStore(); + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = 'not-registered'; + const second = getRunnerConfigStore(); + + expect(first).toBe(store); + expect(second).toBe(store); + expect(createAwsSsmRunnerConfigStoreMock).toHaveBeenCalledOnce(); + }); + + it('selects again after the test reset', () => { + const firstStore = stubStore(); + expect(getRunnerConfigStore()).toBe(firstStore); + + const secondStore = { create: vi.fn(), houseKeeper: vi.fn() } satisfies RunnerConfigStore; + createAwsSsmRunnerConfigStoreMock.mockReturnValue(secondStore); + resetRunnerConfigStore(); + + expect(getRunnerConfigStore()).toBe(secondStore); + expect(createAwsSsmRunnerConfigStoreMock).toHaveBeenCalledTimes(2); + }); +}); + +function setProvider(provider: string | undefined): void { + if (provider === undefined) { + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + } else { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + } +} + +function stubStore(): RunnerConfigStore { + const store = { create: vi.fn(), houseKeeper: vi.fn() } satisfies RunnerConfigStore; + createAwsSsmRunnerConfigStoreMock.mockReturnValue(store); + return store; +} + +function stubDynamoDbStore(): RunnerConfigStore { + const store = { create: vi.fn(), houseKeeper: vi.fn() } satisfies RunnerConfigStore; + createAwsDynamoDbRunnerConfigStoreMock.mockReturnValue(store); + return store; +} diff --git a/lambdas/libs/storage-providers/runner-config.ts b/lambdas/libs/storage-providers/runner-config.ts new file mode 100644 index 0000000000..d4d3998d15 --- /dev/null +++ b/lambdas/libs/storage-providers/runner-config.ts @@ -0,0 +1,25 @@ +import { createAwsDynamoDbRunnerConfigStore } from './aws/dynamodb/runner-config-store'; +import { createAwsSsmRunnerConfigStore } from './aws/ssm/runner-config-store'; +import type { RunnerConfigStore } from './core'; +import type {} from './environment'; +import { resolveRunnerConfigStorageProvider, type RunnerConfigStorageProvider } from './provider'; + +type RunnerConfigStoreFactory = () => RunnerConfigStore; + +const providerFactories = { + aws_ssm: createAwsSsmRunnerConfigStore, + aws_dynamodb: createAwsDynamoDbRunnerConfigStore, +} as const satisfies Record; + +let runnerConfigStore: RunnerConfigStore | undefined; + +export function getRunnerConfigStore(): RunnerConfigStore { + runnerConfigStore ??= + providerFactories[resolveRunnerConfigStorageProvider(process.env.RUNNER_CONFIG_STORAGE_PROVIDER)](); + return runnerConfigStore; +} + +// Test-only reset for cases that need to exercise first-use environment selection. +export function resetRunnerConfigStore(): void { + runnerConfigStore = undefined; +} diff --git a/lambdas/libs/storage-providers/runner-group-cache.test.ts b/lambdas/libs/storage-providers/runner-group-cache.test.ts new file mode 100644 index 0000000000..029e33be8a --- /dev/null +++ b/lambdas/libs/storage-providers/runner-group-cache.test.ts @@ -0,0 +1,106 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAwsDynamoDbRunnerGroupCacheStore } from './aws/dynamodb/runner-group-cache-store'; +import { createAwsSsmRunnerGroupCacheStore } from './aws/ssm/runner-group-cache-store'; +import type { RunnerGroupCacheStore } from './core'; +import { getRunnerGroupCacheStore, resetRunnerGroupCacheStore } from './runner-group-cache'; + +vi.mock('./aws/dynamodb/runner-group-cache-store', () => ({ + createAwsDynamoDbRunnerGroupCacheStore: vi.fn(), +})); +vi.mock('./aws/ssm/runner-group-cache-store', () => ({ + createAwsSsmRunnerGroupCacheStore: vi.fn(), +})); + +const createAwsDynamoDbRunnerGroupCacheStoreMock = vi.mocked(createAwsDynamoDbRunnerGroupCacheStore); +const createAwsSsmRunnerGroupCacheStoreMock = vi.mocked(createAwsSsmRunnerGroupCacheStore); +const cleanEnv = process.env; + +describe('runner group cache store selection', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env = { ...cleanEnv }; + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + resetRunnerGroupCacheStore(); + }); + + it.each([undefined, '', ' '])('uses aws_ssm for default selector input %j', (provider) => { + setProvider(provider); + const store = stubStore(); + + expect(getRunnerGroupCacheStore()).toBe(store); + expect(createAwsSsmRunnerGroupCacheStoreMock).toHaveBeenCalledOnce(); + expect(createAwsDynamoDbRunnerGroupCacheStoreMock).not.toHaveBeenCalled(); + }); + + it.each(['aws_ssm', ' AWS_SSM '])('uses aws_ssm for explicit selector input %j', (provider) => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + const store = stubStore(); + + expect(getRunnerGroupCacheStore()).toBe(store); + expect(createAwsSsmRunnerGroupCacheStoreMock).toHaveBeenCalledOnce(); + expect(createAwsDynamoDbRunnerGroupCacheStoreMock).not.toHaveBeenCalled(); + }); + + it.each(['aws_dynamodb', ' AWS_DYNAMODB '])('uses aws_dynamodb for explicit selector input %j', (provider) => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + const store = stubDynamoDbStore(); + + expect(getRunnerGroupCacheStore()).toBe(store); + expect(createAwsDynamoDbRunnerGroupCacheStoreMock).toHaveBeenCalledOnce(); + expect(createAwsSsmRunnerGroupCacheStoreMock).not.toHaveBeenCalled(); + }); + + it('rejects an unsupported provider on first use', () => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = 'not-registered'; + + expect(() => getRunnerGroupCacheStore()).toThrow("Unsupported runner config storage provider 'not-registered'"); + expect(createAwsSsmRunnerGroupCacheStoreMock).not.toHaveBeenCalled(); + expect(createAwsDynamoDbRunnerGroupCacheStoreMock).not.toHaveBeenCalled(); + }); + + it('selects lazily and caches the created store', () => { + const store = stubStore(); + + expect(createAwsSsmRunnerGroupCacheStoreMock).not.toHaveBeenCalled(); + const first = getRunnerGroupCacheStore(); + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = 'not-registered'; + const second = getRunnerGroupCacheStore(); + + expect(first).toBe(store); + expect(second).toBe(store); + expect(createAwsSsmRunnerGroupCacheStoreMock).toHaveBeenCalledOnce(); + }); + + it('selects again after the test reset', () => { + const firstStore = stubStore(); + expect(getRunnerGroupCacheStore()).toBe(firstStore); + + const secondStore = { get: vi.fn(), create: vi.fn() } satisfies RunnerGroupCacheStore; + createAwsSsmRunnerGroupCacheStoreMock.mockReturnValue(secondStore); + resetRunnerGroupCacheStore(); + + expect(getRunnerGroupCacheStore()).toBe(secondStore); + expect(createAwsSsmRunnerGroupCacheStoreMock).toHaveBeenCalledTimes(2); + }); +}); + +function setProvider(provider: string | undefined): void { + if (provider === undefined) { + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + } else { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + } +} + +function stubStore(): RunnerGroupCacheStore { + const store = { get: vi.fn(), create: vi.fn() } satisfies RunnerGroupCacheStore; + createAwsSsmRunnerGroupCacheStoreMock.mockReturnValue(store); + return store; +} + +function stubDynamoDbStore(): RunnerGroupCacheStore { + const store = { get: vi.fn(), create: vi.fn() } satisfies RunnerGroupCacheStore; + createAwsDynamoDbRunnerGroupCacheStoreMock.mockReturnValue(store); + return store; +} diff --git a/lambdas/libs/storage-providers/runner-group-cache.ts b/lambdas/libs/storage-providers/runner-group-cache.ts new file mode 100644 index 0000000000..3a91437755 --- /dev/null +++ b/lambdas/libs/storage-providers/runner-group-cache.ts @@ -0,0 +1,25 @@ +import { createAwsDynamoDbRunnerGroupCacheStore } from './aws/dynamodb/runner-group-cache-store'; +import { createAwsSsmRunnerGroupCacheStore } from './aws/ssm/runner-group-cache-store'; +import type { RunnerGroupCacheStore } from './core'; +import type {} from './environment'; +import { resolveRunnerConfigStorageProvider, type RunnerConfigStorageProvider } from './provider'; + +type RunnerGroupCacheStoreFactory = () => RunnerGroupCacheStore; + +const providerFactories = { + aws_ssm: createAwsSsmRunnerGroupCacheStore, + aws_dynamodb: createAwsDynamoDbRunnerGroupCacheStore, +} as const satisfies Record; + +let runnerGroupCacheStore: RunnerGroupCacheStore | undefined; + +export function getRunnerGroupCacheStore(): RunnerGroupCacheStore { + runnerGroupCacheStore ??= + providerFactories[resolveRunnerConfigStorageProvider(process.env.RUNNER_CONFIG_STORAGE_PROVIDER)](); + return runnerGroupCacheStore; +} + +// Test-only reset for cases that need to exercise first-use environment selection. +export function resetRunnerGroupCacheStore(): void { + runnerGroupCacheStore = undefined; +} diff --git a/lambdas/libs/storage-providers/runner-matcher-config.test.ts b/lambdas/libs/storage-providers/runner-matcher-config.test.ts index 0dd7f42ad3..981ff28567 100644 --- a/lambdas/libs/storage-providers/runner-matcher-config.test.ts +++ b/lambdas/libs/storage-providers/runner-matcher-config.test.ts @@ -1,13 +1,18 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { createAwsDynamoDbRunnerMatcherConfigStore } from './aws/dynamodb/runner-matcher-config-store'; import { createAwsSsmRunnerMatcherConfigStore } from './aws/ssm/runner-matcher-config-store'; import type { RunnerMatcherConfigStore } from './core'; import { getRunnerMatcherConfigStore, resetRunnerMatcherConfigStore } from './runner-matcher-config'; +vi.mock('./aws/dynamodb/runner-matcher-config-store', () => ({ + createAwsDynamoDbRunnerMatcherConfigStore: vi.fn(), +})); vi.mock('./aws/ssm/runner-matcher-config-store', () => ({ createAwsSsmRunnerMatcherConfigStore: vi.fn(), })); +const createAwsDynamoDbRunnerMatcherConfigStoreMock = vi.mocked(createAwsDynamoDbRunnerMatcherConfigStore); const createAwsSsmRunnerMatcherConfigStoreMock = vi.mocked(createAwsSsmRunnerMatcherConfigStore); const cleanEnv = process.env; @@ -25,6 +30,7 @@ describe('runner matcher config store selection', () => { expect(getRunnerMatcherConfigStore()).toBe(store); expect(createAwsSsmRunnerMatcherConfigStoreMock).toHaveBeenCalledOnce(); + expect(createAwsDynamoDbRunnerMatcherConfigStoreMock).not.toHaveBeenCalled(); }); it.each(['aws_ssm', ' AWS_SSM '])('uses aws_ssm for explicit selector input %j', (provider) => { @@ -33,6 +39,16 @@ describe('runner matcher config store selection', () => { expect(getRunnerMatcherConfigStore()).toBe(store); expect(createAwsSsmRunnerMatcherConfigStoreMock).toHaveBeenCalledOnce(); + expect(createAwsDynamoDbRunnerMatcherConfigStoreMock).not.toHaveBeenCalled(); + }); + + it.each(['aws_dynamodb', ' AWS_DYNAMODB '])('uses aws_dynamodb for explicit selector input %j', (provider) => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + const store = stubDynamoDbStore(); + + expect(getRunnerMatcherConfigStore()).toBe(store); + expect(createAwsDynamoDbRunnerMatcherConfigStoreMock).toHaveBeenCalledOnce(); + expect(createAwsSsmRunnerMatcherConfigStoreMock).not.toHaveBeenCalled(); }); it('rejects an unsupported provider on first use', () => { @@ -40,6 +56,7 @@ describe('runner matcher config store selection', () => { expect(() => getRunnerMatcherConfigStore()).toThrow("Unsupported runner config storage provider 'not-registered'"); expect(createAwsSsmRunnerMatcherConfigStoreMock).not.toHaveBeenCalled(); + expect(createAwsDynamoDbRunnerMatcherConfigStoreMock).not.toHaveBeenCalled(); }); it('selects lazily and caches the created store', () => { @@ -81,3 +98,9 @@ function stubStore(): RunnerMatcherConfigStore { createAwsSsmRunnerMatcherConfigStoreMock.mockReturnValue(store); return store; } + +function stubDynamoDbStore(): RunnerMatcherConfigStore { + const store = { get: vi.fn() } satisfies RunnerMatcherConfigStore; + createAwsDynamoDbRunnerMatcherConfigStoreMock.mockReturnValue(store); + return store; +} diff --git a/lambdas/libs/storage-providers/runner-matcher-config.ts b/lambdas/libs/storage-providers/runner-matcher-config.ts index 6d56d49754..c121fc2aaf 100644 --- a/lambdas/libs/storage-providers/runner-matcher-config.ts +++ b/lambdas/libs/storage-providers/runner-matcher-config.ts @@ -1,3 +1,4 @@ +import { createAwsDynamoDbRunnerMatcherConfigStore } from './aws/dynamodb/runner-matcher-config-store'; import { createAwsSsmRunnerMatcherConfigStore } from './aws/ssm/runner-matcher-config-store'; import type { RunnerMatcherConfigStore } from './core'; import type {} from './environment'; @@ -7,6 +8,7 @@ type RunnerMatcherConfigStoreFactory = () => RunnerMatcherConfigStore; const providerFactories = { aws_ssm: createAwsSsmRunnerMatcherConfigStore, + aws_dynamodb: createAwsDynamoDbRunnerMatcherConfigStore, } as const satisfies Record; let runnerMatcherConfigStore: RunnerMatcherConfigStore | undefined; diff --git a/lambdas/libs/storage-providers/runner-state.test.ts b/lambdas/libs/storage-providers/runner-state.test.ts new file mode 100644 index 0000000000..0d997614e7 --- /dev/null +++ b/lambdas/libs/storage-providers/runner-state.test.ts @@ -0,0 +1,89 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAwsDynamoDbRunnerStateStore } from './aws/dynamodb/runner-state-store'; +import type { RunnerStateStore } from './core'; +import { getRunnerStateStore, resetRunnerStateStore } from './runner-state'; + +vi.mock('./aws/dynamodb/runner-state-store', () => ({ + createAwsDynamoDbRunnerStateStore: vi.fn(), +})); + +const createAwsDynamoDbRunnerStateStoreMock = vi.mocked(createAwsDynamoDbRunnerStateStore); +const cleanEnv = process.env; + +describe('runner state store selection', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env = { ...cleanEnv }; + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + resetRunnerStateStore(); + }); + + it.each([undefined, '', ' ', 'aws_ssm', ' AWS_SSM '])( + 'returns no inventory capability for aws_ssm selector input %j', + (provider) => { + setProvider(provider); + + expect(getRunnerStateStore()).toBeUndefined(); + expect(createAwsDynamoDbRunnerStateStoreMock).not.toHaveBeenCalled(); + }, + ); + + it.each(['aws_dynamodb', ' AWS_DYNAMODB '])('creates the DynamoDB inventory for selector input %j', (provider) => { + setProvider(provider); + const store = stubStore(); + + expect(getRunnerStateStore()).toBe(store); + expect(createAwsDynamoDbRunnerStateStoreMock).toHaveBeenCalledOnce(); + }); + + it('rejects an unsupported provider before creating a store', () => { + setProvider('not-registered'); + + expect(() => getRunnerStateStore()).toThrow("Unsupported runner config storage provider 'not-registered'"); + expect(createAwsDynamoDbRunnerStateStoreMock).not.toHaveBeenCalled(); + }); + + it('caches the DynamoDB store until reset', () => { + setProvider('aws_dynamodb'); + const firstStore = stubStore(); + + expect(getRunnerStateStore()).toBe(firstStore); + expect(getRunnerStateStore()).toBe(firstStore); + expect(createAwsDynamoDbRunnerStateStoreMock).toHaveBeenCalledOnce(); + + const secondStore = createStubStore(); + createAwsDynamoDbRunnerStateStoreMock.mockReturnValue(secondStore); + resetRunnerStateStore(); + expect(getRunnerStateStore()).toBe(secondStore); + expect(createAwsDynamoDbRunnerStateStoreMock).toHaveBeenCalledTimes(2); + }); +}); + +function setProvider(provider: string | undefined): void { + if (provider === undefined) { + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + } else { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + } +} + +function stubStore(): RunnerStateStore { + const store = createStubStore(); + createAwsDynamoDbRunnerStateStoreMock.mockReturnValue(store); + return store; +} + +function createStubStore(): RunnerStateStore { + return { + create: vi.fn(), + recordGitHubIdentity: vi.fn(), + activate: vi.fn(), + list: vi.fn(), + markOrphan: vi.fn(), + unmarkOrphan: vi.fn(), + beginTermination: vi.fn(), + cancelTermination: vi.fn(), + delete: vi.fn(), + }; +} diff --git a/lambdas/libs/storage-providers/runner-state.ts b/lambdas/libs/storage-providers/runner-state.ts new file mode 100644 index 0000000000..e09a09d2b6 --- /dev/null +++ b/lambdas/libs/storage-providers/runner-state.ts @@ -0,0 +1,25 @@ +import { createAwsDynamoDbRunnerStateStore } from './aws/dynamodb/runner-state-store'; +import type { RunnerStateStore } from './core'; +import type {} from './environment'; +import { resolveRunnerConfigStorageProvider } from './provider'; + +let runnerStateStore: RunnerStateStore | undefined; + +export function getRunnerStateStore(): RunnerStateStore | undefined { + if (runnerStateStore) { + return runnerStateStore; + } + + const provider = resolveRunnerConfigStorageProvider(process.env.RUNNER_CONFIG_STORAGE_PROVIDER); + if (provider !== 'aws_dynamodb') { + return undefined; + } + + runnerStateStore = createAwsDynamoDbRunnerStateStore(); + return runnerStateStore; +} + +// Test-only reset for cases that need to exercise first-use environment selection. +export function resetRunnerStateStore(): void { + runnerStateStore = undefined; +} diff --git a/lambdas/libs/storage-providers/vitest.config.ts b/lambdas/libs/storage-providers/vitest.config.ts index a009dab1bf..300d39fbf6 100644 --- a/lambdas/libs/storage-providers/vitest.config.ts +++ b/lambdas/libs/storage-providers/vitest.config.ts @@ -15,6 +15,7 @@ export default mergeConfig(defaultConfig, { 'provider.ts', 'github-webhook-secret.ts', 'runner-matcher-config.ts', + 'runner-state.ts', 'core/**/*.ts', 'aws/**/*.ts', ], diff --git a/lambdas/yarn.lock b/lambdas/yarn.lock index 17e99e638d..2cbfaa1638 100644 --- a/lambdas/yarn.lock +++ b/lambdas/yarn.lock @@ -207,6 +207,7 @@ __metadata: dependencies: "@aws-github-runner/aws-powertools-util": "npm:*" "@aws-github-runner/aws-ssm-util": "npm:*" + "@aws-sdk/client-dynamodb": "npm:^3.1009.0" "@aws-sdk/client-ssm": "npm:^3.1009.0" aws-sdk-client-mock: "npm:^4.1.0" aws-sdk-client-mock-jest: "npm:^4.1.0" @@ -346,6 +347,24 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/client-dynamodb@npm:^3.1009.0": + version: 3.1124.0 + resolution: "@aws-sdk/client-dynamodb@npm:3.1124.0" + dependencies: + "@aws-sdk/core": "npm:^3.977.9" + "@aws-sdk/credential-provider-node": "npm:^3.972.82" + "@aws-sdk/dynamodb-codec": "npm:^3.973.44" + "@aws-sdk/middleware-endpoint-discovery": "npm:^3.972.30" + "@aws-sdk/types": "npm:^3.974.5" + "@smithy/core": "npm:^3.33.3" + "@smithy/fetch-http-handler": "npm:^5.7.2" + "@smithy/node-http-handler": "npm:^4.11.3" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/d0875ae63685617db38ea4033a5fd5ea229f548595561007c56baa6f50704abe435a222a5cd62eda89fdfa5a96fadaf1aa98e234d694ecb15281bfe92226d3b5 + languageName: node + linkType: hard + "@aws-sdk/client-ec2@npm:^3.1009.0": version: 3.1014.0 resolution: "@aws-sdk/client-ec2@npm:3.1014.0" @@ -624,6 +643,22 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/core@npm:^3.977.9": + version: 3.977.9 + resolution: "@aws-sdk/core@npm:3.977.9" + dependencies: + "@aws-sdk/types": "npm:^3.974.5" + "@aws-sdk/xml-builder": "npm:^3.972.40" + "@aws/lambda-invoke-store": "npm:^0.3.0" + "@smithy/core": "npm:^3.33.3" + "@smithy/signature-v4": "npm:^5.6.12" + "@smithy/types": "npm:^4.17.2" + bowser: "npm:^2.11.0" + tslib: "npm:^2.6.2" + checksum: 10c0/9118a8d05b7c27fe55fb5e793840193d1b311b6c0c2958e591bcada391a71783536ed9c8f4913cc9c4b7258dba650d625ed782669a12875348544c33f35aa8d6 + languageName: node + linkType: hard + "@aws-sdk/crc64-nvme@npm:^3.972.5": version: 3.972.5 resolution: "@aws-sdk/crc64-nvme@npm:3.972.5" @@ -647,6 +682,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-env@npm:^3.972.70": + version: 3.972.70 + resolution: "@aws-sdk/credential-provider-env@npm:3.972.70" + dependencies: + "@aws-sdk/core": "npm:^3.977.9" + "@aws-sdk/types": "npm:^3.974.5" + "@smithy/core": "npm:^3.33.3" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/f5d8f3f1021a83911f617dd6b36277486d0bdb174c107c4f7776a7e2c609a101aa03ada1cca9d97229ac79aea7797ec6392fe449029b87f7cd013360592bddf3 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-http@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-http@npm:3.972.23" @@ -665,6 +713,21 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-http@npm:^3.972.72": + version: 3.972.72 + resolution: "@aws-sdk/credential-provider-http@npm:3.972.72" + dependencies: + "@aws-sdk/core": "npm:^3.977.9" + "@aws-sdk/types": "npm:^3.974.5" + "@smithy/core": "npm:^3.33.3" + "@smithy/fetch-http-handler": "npm:^5.7.2" + "@smithy/node-http-handler": "npm:^4.11.3" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/aaa67dc42ce00713d92f931c620e36cb199533e6f1f892a6be76553986c86977442924e76f3743732a7b65e6f6777271789a6edbc70d5a86e0a9b6b5a3ef25f3 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-ini@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-ini@npm:3.972.23" @@ -687,6 +750,27 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-ini@npm:^3.973.15": + version: 3.973.15 + resolution: "@aws-sdk/credential-provider-ini@npm:3.973.15" + dependencies: + "@aws-sdk/core": "npm:^3.977.9" + "@aws-sdk/credential-provider-env": "npm:^3.972.70" + "@aws-sdk/credential-provider-http": "npm:^3.972.72" + "@aws-sdk/credential-provider-login": "npm:^3.972.77" + "@aws-sdk/credential-provider-process": "npm:^3.972.70" + "@aws-sdk/credential-provider-sso": "npm:^3.973.14" + "@aws-sdk/credential-provider-web-identity": "npm:^3.972.76" + "@aws-sdk/nested-clients": "npm:^3.997.44" + "@aws-sdk/types": "npm:^3.974.5" + "@smithy/core": "npm:^3.33.3" + "@smithy/credential-provider-imds": "npm:^4.4.16" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/1fe5f8c84cb9877a62084f9cbedebaed605ecc34891c44254d979ccd1282a5e164840d6dbb340773c311c028568f9ea642649bebf4b38cebb2d6d0039ad9de12 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-login@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-login@npm:3.972.23" @@ -703,6 +787,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-login@npm:^3.972.77": + version: 3.972.77 + resolution: "@aws-sdk/credential-provider-login@npm:3.972.77" + dependencies: + "@aws-sdk/core": "npm:^3.977.9" + "@aws-sdk/nested-clients": "npm:^3.997.44" + "@aws-sdk/types": "npm:^3.974.5" + "@smithy/core": "npm:^3.33.3" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/9be54fdf406325d462abdc0a2a182ba1ed39009fa18b60035705e6c3fae92dfe1afe5dba24a4fa3c8376f7cd2e1dad7a91152909f3b96dde0b215a88dfcd5330 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-node@npm:^3.972.24": version: 3.972.24 resolution: "@aws-sdk/credential-provider-node@npm:3.972.24" @@ -723,6 +821,25 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-node@npm:^3.972.82": + version: 3.972.82 + resolution: "@aws-sdk/credential-provider-node@npm:3.972.82" + dependencies: + "@aws-sdk/credential-provider-env": "npm:^3.972.70" + "@aws-sdk/credential-provider-http": "npm:^3.972.72" + "@aws-sdk/credential-provider-ini": "npm:^3.973.15" + "@aws-sdk/credential-provider-process": "npm:^3.972.70" + "@aws-sdk/credential-provider-sso": "npm:^3.973.14" + "@aws-sdk/credential-provider-web-identity": "npm:^3.972.76" + "@aws-sdk/types": "npm:^3.974.5" + "@smithy/core": "npm:^3.33.3" + "@smithy/credential-provider-imds": "npm:^4.4.16" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/c79c2381c1e76fecadb5adf9a8623946d819b20bfc0eaa236a384115027cdea3fe1f0a294c75edbb6aee1a0ccbbc4ec606c2eee8e1c4d720ed2ef281aa1175c8 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-process@npm:^3.972.21": version: 3.972.21 resolution: "@aws-sdk/credential-provider-process@npm:3.972.21" @@ -737,6 +854,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-process@npm:^3.972.70": + version: 3.972.70 + resolution: "@aws-sdk/credential-provider-process@npm:3.972.70" + dependencies: + "@aws-sdk/core": "npm:^3.977.9" + "@aws-sdk/types": "npm:^3.974.5" + "@smithy/core": "npm:^3.33.3" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/87ea5f575f611461a538312b6c502baf86ce33eef6974bb2f9e85bd32147cb8f6b518e80f5a5d8ef371fceb03d517f4d3b6c07ddeb7f15d5a2f56c6dcbf8e8ba + languageName: node + linkType: hard + "@aws-sdk/credential-provider-sso@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-sso@npm:3.972.23" @@ -753,6 +883,21 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-sso@npm:^3.973.14": + version: 3.973.14 + resolution: "@aws-sdk/credential-provider-sso@npm:3.973.14" + dependencies: + "@aws-sdk/core": "npm:^3.977.9" + "@aws-sdk/nested-clients": "npm:^3.997.44" + "@aws-sdk/token-providers": "npm:3.1116.0" + "@aws-sdk/types": "npm:^3.974.5" + "@smithy/core": "npm:^3.33.3" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/12132e57c07277b4c115c835bb7a14b2a62e4f30a69998eca501a171d46be469439b3e7b33c970165362bd261b8689389dd06ba0a959657d97330f3e5172ec52 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-web-identity@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-web-identity@npm:3.972.23" @@ -768,6 +913,42 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-web-identity@npm:^3.972.76": + version: 3.972.76 + resolution: "@aws-sdk/credential-provider-web-identity@npm:3.972.76" + dependencies: + "@aws-sdk/core": "npm:^3.977.9" + "@aws-sdk/nested-clients": "npm:^3.997.44" + "@aws-sdk/types": "npm:^3.974.5" + "@smithy/core": "npm:^3.33.3" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/5efcf6a4b10e49b5b3f9bf76b638ff0b0ce84d26126197f0d28cecbd01458f5d84a074a4cdc8817a7310a940a36a8ad164c1531eb031713bd1485d52bef8c877 + languageName: node + linkType: hard + +"@aws-sdk/dynamodb-codec@npm:^3.973.44": + version: 3.973.44 + resolution: "@aws-sdk/dynamodb-codec@npm:3.973.44" + dependencies: + "@aws-sdk/core": "npm:^3.977.9" + "@smithy/core": "npm:^3.33.3" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/6c7d9c68ac87bf8806b978b3200cfaa81f373ccba466ed43306416ff8a9d794214b5a7c03695ebff83ea818de99d90f9812e41fb0742c5bac0b3fdd308833299 + languageName: node + linkType: hard + +"@aws-sdk/endpoint-cache@npm:^3.972.11": + version: 3.972.11 + resolution: "@aws-sdk/endpoint-cache@npm:3.972.11" + dependencies: + mnemonist: "npm:0.38.3" + tslib: "npm:^2.6.2" + checksum: 10c0/8f3c039fb2dd8e434cd2992bce2a840e6f6470be5ae6d71c762f6cb842149fb60d5566ca470319d5abcf90b115d2c1a0114ed3328afbfc0abed4cbcd52ad03a1 + languageName: node + linkType: hard + "@aws-sdk/lib-storage@npm:^3.1009.0": version: 3.1014.0 resolution: "@aws-sdk/lib-storage@npm:3.1014.0" @@ -800,6 +981,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/middleware-endpoint-discovery@npm:^3.972.30": + version: 3.972.30 + resolution: "@aws-sdk/middleware-endpoint-discovery@npm:3.972.30" + dependencies: + "@aws-sdk/endpoint-cache": "npm:^3.972.11" + "@aws-sdk/types": "npm:^3.974.5" + "@smithy/core": "npm:^3.33.3" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/5655eccf7293f481f45a57359a7d7737ff7345cae3602ae5cb669653f7bc1503f61ce837f11d8bfe399726ae189503cfdc2c21d198816f5acc2d238b7b3476e2 + languageName: node + linkType: hard + "@aws-sdk/middleware-expect-continue@npm:^3.972.8": version: 3.972.8 resolution: "@aws-sdk/middleware-expect-continue@npm:3.972.8" @@ -1006,6 +1200,22 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/nested-clients@npm:^3.997.44": + version: 3.997.44 + resolution: "@aws-sdk/nested-clients@npm:3.997.44" + dependencies: + "@aws-sdk/core": "npm:^3.977.9" + "@aws-sdk/signature-v4-multi-region": "npm:^3.996.46" + "@aws-sdk/types": "npm:^3.974.5" + "@smithy/core": "npm:^3.33.3" + "@smithy/fetch-http-handler": "npm:^5.7.2" + "@smithy/node-http-handler": "npm:^4.11.3" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/947c2049a69a02399f600bd448f34bcaaf3a97b5bbb9b43e9ce932e84d4e9c3131687d330b9c48391da4365582890dd72bf4b8dfafe0693c041e48aa0f5c972d + languageName: node + linkType: hard + "@aws-sdk/region-config-resolver@npm:^3.972.9": version: 3.972.9 resolution: "@aws-sdk/region-config-resolver@npm:3.972.9" @@ -1033,6 +1243,18 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/signature-v4-multi-region@npm:^3.996.46": + version: 3.996.46 + resolution: "@aws-sdk/signature-v4-multi-region@npm:3.996.46" + dependencies: + "@aws-sdk/types": "npm:^3.974.5" + "@smithy/signature-v4": "npm:^5.6.12" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/069dfb7a95663cad2e0aec1d87df8a800abad33cb49dfbe9412dad2d63ab6350328c6165166b12d50e609d8dbe5a5536aa6b1101be4d91584fbca76b2fea4d00 + languageName: node + linkType: hard + "@aws-sdk/token-providers@npm:3.1014.0": version: 3.1014.0 resolution: "@aws-sdk/token-providers@npm:3.1014.0" @@ -1048,6 +1270,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/token-providers@npm:3.1116.0": + version: 3.1116.0 + resolution: "@aws-sdk/token-providers@npm:3.1116.0" + dependencies: + "@aws-sdk/core": "npm:^3.977.9" + "@aws-sdk/nested-clients": "npm:^3.997.44" + "@aws-sdk/types": "npm:^3.974.5" + "@smithy/core": "npm:^3.33.3" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/e720401f6b6d5682984cf1927d3e4c86663b750086bac3b58827a3b9b8585c8b06fb93af82a03a40d5a8ca48c89c0f47c1ac48ea81822c52ddecc3e36ef7cf17 + languageName: node + linkType: hard + "@aws-sdk/types@npm:^3.222.0, @aws-sdk/types@npm:^3.4.1, @aws-sdk/types@npm:^3.973.6": version: 3.973.6 resolution: "@aws-sdk/types@npm:3.973.6" @@ -1058,6 +1294,16 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/types@npm:^3.974.5": + version: 3.974.5 + resolution: "@aws-sdk/types@npm:3.974.5" + dependencies: + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/803aaaa1c0675dcb564803993f3c47d96302fad461af8af80e75afc40c72228ebf669593c18e44ca09fc5acf0d1bd25966261de07844d8f11ad82aa2650252d0 + languageName: node + linkType: hard + "@aws-sdk/util-arn-parser@npm:^3.972.3": version: 3.972.3 resolution: "@aws-sdk/util-arn-parser@npm:3.972.3" @@ -1143,6 +1389,16 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/xml-builder@npm:^3.972.40": + version: 3.972.40 + resolution: "@aws-sdk/xml-builder@npm:3.972.40" + dependencies: + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/5b06fa0466b5ddb0e33138dc16f6a30e11e52fc5ea71f3ed72af7b24621d29c9e8103df3eed9c4f14cef50f4d345dd9e7021242a8c155a67869ee4ce79982bb4 + languageName: node + linkType: hard + "@aws/lambda-invoke-store@npm:0.2.3, @aws/lambda-invoke-store@npm:^0.2.2": version: 0.2.3 resolution: "@aws/lambda-invoke-store@npm:0.2.3" @@ -1150,6 +1406,13 @@ __metadata: languageName: node linkType: hard +"@aws/lambda-invoke-store@npm:^0.3.0": + version: 0.3.0 + resolution: "@aws/lambda-invoke-store@npm:0.3.0" + checksum: 10c0/b4a2e6b3b5397bc606053e64270d26dc5c886336f88a98cad587b1592eec17058f8fb172f1827a9f0e591f3595cf8f01575c8c9b36cde38c06456f8a65204046 + languageName: node + linkType: hard + "@babel/code-frame@npm:^7.0.0, @babel/code-frame@npm:^7.12.13, @babel/code-frame@npm:^7.28.6, @babel/code-frame@npm:^7.29.0": version: 7.29.0 resolution: "@babel/code-frame@npm:7.29.0" @@ -4570,6 +4833,16 @@ __metadata: languageName: node linkType: hard +"@smithy/core@npm:^3.33.2, @smithy/core@npm:^3.33.3": + version: 3.33.3 + resolution: "@smithy/core@npm:3.33.3" + dependencies: + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/57c5c6c1834d84eddfff905932350973afa20e1006024b82d6599af4c8d1e75b243e06b93c998acc95946399f1342ddc298c6941e0030384e6541ceec4007376 + languageName: node + linkType: hard + "@smithy/credential-provider-imds@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/credential-provider-imds@npm:4.2.12" @@ -4583,6 +4856,17 @@ __metadata: languageName: node linkType: hard +"@smithy/credential-provider-imds@npm:^4.4.16": + version: 4.5.2 + resolution: "@smithy/credential-provider-imds@npm:4.5.2" + dependencies: + "@smithy/core": "npm:^3.33.2" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/d5481a7797a1f485849d92f6c188cfb9411736ae2469e27394b863e107dd1024266baa8e3a62cfc3f75b3abfd84b3c389955a0e96b64c4ad90462c70bbe66ab0 + languageName: node + linkType: hard + "@smithy/eventstream-codec@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/eventstream-codec@npm:4.2.12" @@ -4651,6 +4935,17 @@ __metadata: languageName: node linkType: hard +"@smithy/fetch-http-handler@npm:^5.7.2": + version: 5.7.2 + resolution: "@smithy/fetch-http-handler@npm:5.7.2" + dependencies: + "@smithy/core": "npm:^3.33.2" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/9f4541cb743a5207f33d6abd135642a5310cc24f0ed5d46836bf3692901110216f64e09df977488a652762332156fe914b934c780f6a95ef8b1c7d3235c99c7f + languageName: node + linkType: hard + "@smithy/hash-blob-browser@npm:^4.2.13": version: 4.2.13 resolution: "@smithy/hash-blob-browser@npm:4.2.13" @@ -4803,6 +5098,17 @@ __metadata: languageName: node linkType: hard +"@smithy/node-http-handler@npm:^4.11.3": + version: 4.11.3 + resolution: "@smithy/node-http-handler@npm:4.11.3" + dependencies: + "@smithy/core": "npm:^3.33.3" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/b1ec5956281d7ab5a5d1d4ca890b5d9f84f6540bb80c07e5b0eb31426bad27ff03ae56851e95c3fc8efb52ac9d432df86f9735e1be01db7ecdd1559a85683589 + languageName: node + linkType: hard + "@smithy/node-http-handler@npm:^4.5.0": version: 4.5.0 resolution: "@smithy/node-http-handler@npm:4.5.0" @@ -4901,6 +5207,17 @@ __metadata: languageName: node linkType: hard +"@smithy/signature-v4@npm:^5.6.12": + version: 5.7.3 + resolution: "@smithy/signature-v4@npm:5.7.3" + dependencies: + "@smithy/core": "npm:^3.33.3" + "@smithy/types": "npm:^4.17.2" + tslib: "npm:^2.6.2" + checksum: 10c0/6976142320c7c112ee817f5329d0adc45c1ec101a095a70e4afcb7e3fa9f18cd4f544dcbb481e1932fb877197ec9e1037054bacda422cccfd8de7eed55905319 + languageName: node + linkType: hard + "@smithy/smithy-client@npm:^4.12.7": version: 4.12.7 resolution: "@smithy/smithy-client@npm:4.12.7" @@ -4934,6 +5251,15 @@ __metadata: languageName: node linkType: hard +"@smithy/types@npm:^4.17.2": + version: 4.17.2 + resolution: "@smithy/types@npm:4.17.2" + dependencies: + tslib: "npm:^2.6.2" + checksum: 10c0/7a11f38e6dacdf2247c17bcd50a68ce6200a09dc6a7ecd8d563de94855a25c1da45c4d98f586e3c6fe94727ddbbde6d88ff212ac8eabf7d49779902d1a99a715 + languageName: node + linkType: hard + "@smithy/url-parser@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/url-parser@npm:4.2.12" @@ -9069,6 +9395,15 @@ __metadata: languageName: node linkType: hard +"mnemonist@npm:0.38.3": + version: 0.38.3 + resolution: "mnemonist@npm:0.38.3" + dependencies: + obliterator: "npm:^1.6.1" + checksum: 10c0/064aa1ee1a89fce2754423b3617c598fd65bc34311eb3c01dc063976f6b819b073bd23532415cf8c92240157b4c8fbb7ec5d79d717f2bd4fcd95d8131cb23acb + languageName: node + linkType: hard + "moment-timezone@npm:^0.6.0": version: 0.6.0 resolution: "moment-timezone@npm:0.6.0" @@ -9471,6 +9806,13 @@ __metadata: languageName: node linkType: hard +"obliterator@npm:^1.6.1": + version: 1.6.1 + resolution: "obliterator@npm:1.6.1" + checksum: 10c0/5fad57319aae0ef6e34efa640541d41c2dd9790a7ab808f17dcb66c83a81333963fc2dfcfa6e1b62158e5cef6291cdcf15c503ad6c3de54b2227dd4c3d7e1b55 + languageName: node + linkType: hard + "obug@npm:^2.1.1": version: 2.1.1 resolution: "obug@npm:2.1.1" From 9dc0ba785b994b6c7a545a6f2383f9df25341faf Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Tue, 8 Sep 2026 23:46:13 +0200 Subject: [PATCH 4/4] fix(terraform): resolve storage provider rebase conflicts --- .github/workflows/terraform.yml | 14 + .../0002-runner-storage-provider-boundary.md | 196 ++++++++++++ mkdocs.yaml | 1 + modules/compute-providers/aws/ec2/README.md | 1 + .../aws/ec2/control-plane.tf | 6 +- .../aws/ec2/policies-runner.tf | 16 +- .../aws/ec2/runner-config.tf | 12 + .../aws/ec2/runner-instances.tf | 7 +- .../aws/ec2/templates/start-runner-osx.sh | 63 ++++ .../aws/ec2/templates/start-runner.ps1 | 73 +++++ .../aws/ec2/templates/start-runner.sh | 64 ++++ .../aws/ec2/tests/provider.tftest.hcl | 84 +++++- .../compute-providers/aws/ec2/variables.tf | 37 +++ modules/multi-runner/README.md | 3 + .../config.experimental.resolved.tf | 2 + .../config.experimental.translation.tf | 7 + modules/multi-runner/storage-provider.tf | 160 ++++++++++ .../variables.experimental.global.tf | 34 +++ modules/multi-runner/webhook.tf | 7 +- .../orchestration-providers/webhook/README.md | 1 + .../webhook/job-retry.tf | 5 + .../webhook/job-retry/README.md | 1 + .../webhook/job-retry/iam-policies.tf | 32 +- .../webhook/job-retry/job-retry.tf | 21 +- .../webhook/job-retry/variables.tf | 14 + .../orchestration-providers/webhook/main.tf | 13 +- .../orchestration-providers/webhook/pool.tf | 4 + .../webhook/pool/README.md | 1 + .../webhook/pool/iam-policies.tf | 88 +++--- .../webhook/pool/pool.tf | 55 ++-- .../webhook/pool/variables.tf | 14 + .../webhook/scale-runners.tf | 6 + .../webhook/scale-runners/README.md | 1 + .../scale-runners/scale-down-iam-policies.tf | 35 ++- .../webhook/scale-runners/scale-down.tf | 35 +-- .../scale-runners/scale-up-iam-policies.tf | 71 +++-- .../webhook/scale-runners/scale-up.tf | 51 ++-- .../webhook/scale-runners/variables.tf | 26 ++ .../webhook/variables.tf | 42 +++ .../storage-providers/aws/dynamodb/README.md | 59 ++++ .../aws/dynamodb/capabilities.tf | 243 +++++++++++++++ .../aws/dynamodb/config-version.tf | 23 ++ .../storage-providers/aws/dynamodb/items.tf | 56 ++++ .../storage-providers/aws/dynamodb/outputs.tf | 71 +++++ .../storage-providers/aws/dynamodb/tables.tf | 62 ++++ .../aws/dynamodb/tests/provider.tftest.hcl | 284 ++++++++++++++++++ .../aws/dynamodb/variables.tf | 84 ++++++ .../aws/dynamodb/versions.tf | 10 + modules/webhook/README.md | 1 + modules/webhook/direct/README.md | 2 +- modules/webhook/direct/variables.tf | 9 + modules/webhook/direct/webhook.tf | 21 +- modules/webhook/eventbridge/README.md | 2 +- modules/webhook/eventbridge/dispatcher.tf | 19 +- modules/webhook/eventbridge/variables.tf | 21 ++ modules/webhook/eventbridge/webhook.tf | 19 +- modules/webhook/variables.tf | 43 +++ modules/webhook/webhook.tf | 12 +- 58 files changed, 2131 insertions(+), 213 deletions(-) create mode 100644 docs/adr/0002-runner-storage-provider-boundary.md create mode 100644 modules/multi-runner/storage-provider.tf create mode 100644 modules/storage-providers/aws/dynamodb/README.md create mode 100644 modules/storage-providers/aws/dynamodb/capabilities.tf create mode 100644 modules/storage-providers/aws/dynamodb/config-version.tf create mode 100644 modules/storage-providers/aws/dynamodb/items.tf create mode 100644 modules/storage-providers/aws/dynamodb/outputs.tf create mode 100644 modules/storage-providers/aws/dynamodb/tables.tf create mode 100644 modules/storage-providers/aws/dynamodb/tests/provider.tftest.hcl create mode 100644 modules/storage-providers/aws/dynamodb/variables.tf create mode 100644 modules/storage-providers/aws/dynamodb/versions.tf diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml index 6d91e10c66..e222a3c6ee 100644 --- a/.github/workflows/terraform.yml +++ b/.github/workflows/terraform.yml @@ -86,6 +86,13 @@ jobs: "lambda", "multi-runner", "runner-binaries-syncer", + "storage-providers/aws/dynamodb", + "orchestration-providers/webhook", + "orchestration-providers/webhook/job-retry", + "orchestration-providers/webhook/pool", + "orchestration-providers/webhook/scale-runners", + "compute-providers/aws/ec2", + "compute-providers/aws/ec2/trust-policy", "runners", "setup-iam-permissions", "ssm", @@ -215,6 +222,13 @@ jobs: module: - modules/runners - modules/multi-runner + - modules/orchestration-providers/webhook + - modules/orchestration-providers/webhook/job-retry + - modules/orchestration-providers/webhook/pool + - modules/orchestration-providers/webhook/scale-runners + - modules/storage-providers/aws/dynamodb + - modules/compute-providers/aws/ec2 + - modules/compute-providers/aws/ec2/trust-policy defaults: run: working-directory: ${{ matrix.module }} diff --git a/docs/adr/0002-runner-storage-provider-boundary.md b/docs/adr/0002-runner-storage-provider-boundary.md new file mode 100644 index 0000000000..4864388307 --- /dev/null +++ b/docs/adr/0002-runner-storage-provider-boundary.md @@ -0,0 +1,196 @@ +# ADR-0002: Runner Storage Provider Boundary + +## Status + +Proposed + +## Date + +2026-09-08 + +## Context + +Runner operation depends on several kinds of stored data: GitHub App +credentials, webhook secrets, matcher configuration, runner-group mappings, +short-lived runner bootstrap configuration, and runner lifecycle state. The +original implementation stored these values in AWS Systems Manager Parameter +Store (SSM), with parameter names, SecureString handling, cleanup, and IAM +permissions spread across the Lambda and Terraform modules. + +That coupling makes it difficult to provide durable runner inventory and to +change the storage implementation without adding provider-specific branches to +each consumer. It also makes the control plane rely on provider discovery for +runner counts, which is insufficient while a runner is being provisioned or +when a launch succeeds before the rest of its registration flow completes. + +The repository needs a replaceable storage boundary that preserves existing +SSM deployments while allowing an opt-in DynamoDB implementation for the +provider-boundary multi-runner configuration. + +## Decision + +Define provider-neutral storage interfaces for the data used by runner +orchestration and select one storage provider for a deployment. The supported +providers are: + +- `aws_ssm`: the existing default and compatibility path. +- `aws_dynamodb`: the durable, opt-in path for provider-boundary + multi-runner configurations. + +Provider selection is represented by the canonical values `aws_ssm` and +`aws_dynamodb`. An omitted selection resolves to `aws_ssm`. A deployment must +select at most one provider; storage consumers do not silently fall back from +one provider to the other when a credential, permission, or data lookup fails. + +### Provider-neutral contract + +The storage library owns interfaces and provider factories for: + +- GitHub App credentials; +- webhook secrets; +- runner matcher configuration; +- runner configuration creation and one-time consumption; +- runner-group ID caching; and +- runner lifecycle state. + +Control-plane and bootstrap code depends on these interfaces. It does not +construct SSM parameter names or DynamoDB keys. Provider-specific factories are +selected once per Lambda process from the environment and are safe to reuse +within that process. + +Runner bootstrap configuration remains separate from lifecycle state. Bootstrap +configuration contains short-lived or sensitive values and is consumed once; +runner state is durable inventory keyed by the compute resource and records +states such as `provisioning`, `active`, `orphan`, and `terminating`. + +### SSM provider + +The SSM provider retains the established behavior for existing deployments: + +- parameters remain the storage boundary for credentials, secrets, matcher + configuration, runner groups, and runner bootstrap configuration; +- sensitive values use SecureString parameters and existing parameter-store + tagging conventions; +- runner configuration cleanup remains an explicit housekeeper operation; and +- existing stable Terraform inputs continue to translate to the SSM provider. + +SSM does not provide the durable runner-state implementation in this phase. +When state inventory is unavailable, the control plane uses compute-provider +discovery, preserving the existing behavior. + +### DynamoDB provider + +The DynamoDB provider uses two shared tables: + +1. a configuration table for global records and per-runner-entry records; and +2. a runner-state table for durable lifecycle inventory with TTL-based cleanup. + +Records use explicit logical scopes and an `id` so that global data, entry +configuration, runner-group mappings, bootstrap values, and runner state cannot +collide. The provider exposes table names, scopes, TTL settings, and IAM policy +fragments as Terraform capabilities rather than making callers know the table +layout. + +The DynamoDB implementation must enforce the storage contract at the data +operation boundary: + +- one-time bootstrap consumption is conditional and removes the consumed + record; +- lifecycle transitions are conditional so stale workers cannot overwrite a + newer state; +- runner-state records identify the compute provider, compute resource, GitHub + identity when known, owner, runner type, and lifecycle state; and +- IAM policies restrict access with table ARNs and DynamoDB leading-key + conditions. Runner bootstrap access is restricted to the matching compute + resource identity. + +### Terraform capability boundary + +Terraform resolves the selected provider once and passes opaque capabilities to +the webhook orchestration and compute-provider modules. Capabilities include +provider-specific environment variables and IAM policy documents for each +consumer, including the runner bootstrap path. + +The `global_config_storage_provider` input selects the provider for the +provider-boundary configuration. Stable v1 configuration is translated to an +SSM selection, so existing users retain the current backend unless they opt in +to DynamoDB through the provider-boundary configuration. + +The compute provider owns the runner-side capability needed to read bootstrap +configuration. The orchestration provider owns its Lambda resources and +receives only the capabilities it needs. This keeps storage ownership separate +from both compute implementation and orchestration scheduling. + +## Alternatives considered + +### Keep SSM as the only backend + +This preserves the smallest implementation, but does not provide durable +runner inventory or a suitable shared store for the provider-boundary design. + +### Add storage conditionals to every consumer + +This would avoid a factory layer initially, but it would duplicate key +construction, error handling, security rules, and migration behavior across +Lambdas and runner bootstrap code. It would make each new provider more +expensive and easier to implement inconsistently. + +### Use one DynamoDB table for all data + +One table could reduce resource count, but separating configuration from +ephemeral runner state gives the two lifecycles independent TTL, protection, +and access policies. The two-table design also makes accidental access to +runner state from configuration consumers less likely. + +### Migrate existing SSM data automatically + +Automatic migration would require dual writes or a cutover protocol and could +duplicate or lose short-lived bootstrap configuration. Migration is therefore +an explicit operational decision outside provider selection; the default +remains backward compatible with SSM. + +## Consequences + +### Positive + +- Existing stable deployments continue to use SSM without configuration + changes. +- Storage consumers share one provider-neutral contract and do not duplicate + backend logic. +- DynamoDB can provide durable runner inventory and conservative recovery from + launch-before-registration failures. +- Provider-specific IAM conditions and runner bootstrap capabilities can be + reviewed at the Terraform module boundary. +- A future storage provider can implement the same interfaces without changing + orchestration or compute-provider callers. + +### Negative + +- The DynamoDB path adds two tables, TTL behavior, conditional-write logic, + provider-specific IAM, and additional operational cost. +- SSM and DynamoDB have different consistency, cleanup, and failure behavior; + both implementations require provider-specific contract tests. +- Switching an existing deployment does not migrate stored values or active + runner inventory automatically. +- The control plane must retain compute discovery as a recovery source even + when DynamoDB inventory is enabled. + +## Migration and operational rules + +1. Keep `aws_ssm` as the default until a deployment explicitly selects + `aws_dynamodb`. +2. Treat a provider switch as an operational migration with a planned cutover; + do not assume existing SSM records are present in DynamoDB. +3. Keep provider-specific secrets, table names, scopes, and IAM details inside + provider capabilities and environment configuration, not in shared + orchestration code. +4. Add contract tests for every new provider covering reads, writes, + one-time consumption, conditional lifecycle transitions, and authorization + boundaries. + +## References + +- [Storage-provider interfaces and factories](../../lambdas/libs/storage-providers/) +- [DynamoDB storage-provider module](../../modules/storage-providers/aws/dynamodb/) +- [Multi-runner storage-provider composition](../../modules/multi-runner/storage-provider.tf) +- [Compute-provider storage capability contract](../../modules/compute-providers/aws/ec2/variables.tf) diff --git a/mkdocs.yaml b/mkdocs.yaml index 849b9a53dc..e3916290b0 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -60,6 +60,7 @@ nav: - Security: security.md - Architecture decisions: - MiniStack for integration tests: adr/0001-use-ministack-for-terraform-integration-tests.md + - Runner storage provider boundary: adr/0002-runner-storage-provider-boundary.md - Modules: - Runners (main): modules/runners.md - Submodules (public): diff --git a/modules/compute-providers/aws/ec2/README.md b/modules/compute-providers/aws/ec2/README.md index aa2ad83651..7567bd1d2f 100644 --- a/modules/compute-providers/aws/ec2/README.md +++ b/modules/compute-providers/aws/ec2/README.md @@ -67,6 +67,7 @@ No modules. | [prefix](#input\_prefix) | Prefix used to identify resources created for the runner configuration. | `string` | `"github-actions"` | no | | [runner](#input\_runner) | Provider-neutral runner settings consumed by compute providers.

- `os`: Runner operating system. Supported values are `linux`, `osx`, and `windows`.
- `architecture`: Runner distribution architecture.
- `name_prefix`: Prefix added to registered runner names.
- `run_as_root`: Runs the runner service as root.
- `run_as`: Operating-system user used when `run_as_root` is false.
- `hooks.job_started`: Script installed as the runner job-started hook.
- `hooks.job_completed`: Script installed as the runner job-completed hook.
- `iam.role.arn`: Resolved runner-role ARN referenced by provider policies and resources.
- `iam.role.name`: Resolved runner-role name used by provider resources.
- `iam.role.managed`: Whether runner-config manages the resolved runner role.
- `iam.managed_policy_arns`: Common managed-policy ARNs returned with the provider-specific runner policies for attachment by runner-config.
- `iam.path`: IAM path available to provider-managed IAM resources. Null derives the path from `prefix`. | 
object({
    os           = optional(string, "linux")
    architecture = optional(string, "x64")
    name_prefix  = optional(string, "")
    run_as_root  = optional(bool, false)
    run_as       = optional(string, "ec2-user")
    hooks = optional(object({
      job_started   = optional(string, "")
      job_completed = optional(string, "")
    }), {})
    iam = object({
      role = object({
        arn     = string
        name    = string
        managed = optional(bool, true)
      })
      managed_policy_arns = optional(map(string), {})
      path                = optional(string, null)
    })
  })
| n/a | yes | | [ssm](#input\_ssm) | Parameter Store paths and tag scopes available to compute-provider bootstrap resources.

- `paths.root`: Root Parameter Store path for the runner configuration.
- `paths.tokens`: Path segment used for registration tokens and just-in-time configuration.
- `paths.config`: Path segment used for persistent runner and provider configuration.
- `tags`: Shared SSM tags that override module-level `tags`.
- `parameters.tags`: Parameter-specific tags that override module-level and shared SSM tags. | 
object({
    paths = object({
      root   = string
      tokens = string
      config = string
    })
    tags = optional(map(string), {})
    parameters = optional(object({
      tags = optional(map(string), {})
    }), {})
  })
| n/a | yes | +| [storage\_provider](#input\_storage\_provider) | Runner-side storage locator and opaque IAM policy supplied by runner-config. The default preserves the existing SSM bootstrap path. | 
object({
    type = string
    runner = object({
      config_table_name       = optional(string, null)
      runner_state_table_name = optional(string, null)
      scope                   = optional(string, null)
      iam_policy_json         = optional(string, null)
    })
  })
| 
{
  "runner": {
    "config_table_name": null,
    "iam_policy_json": null,
    "runner_state_table_name": null,
    "scope": null
  },
  "type": "aws_ssm"
}
| no | | [tags](#input\_tags) | Base tags available to taggable compute-provider resources. Provider-specific tags override this map within their documented scopes. | `map(string)` | `{}` | no | ## Outputs diff --git a/modules/compute-providers/aws/ec2/control-plane.tf b/modules/compute-providers/aws/ec2/control-plane.tf index 1b25442032..70caee4225 100644 --- a/modules/compute-providers/aws/ec2/control-plane.tf +++ b/modules/compute-providers/aws/ec2/control-plane.tf @@ -191,7 +191,7 @@ data "aws_iam_policy_document" "service_linked_role" { } locals { - scale_up_environment_variables = { + scale_up_environment_variables = merge({ AMI_ID_SSM_PARAMETER_NAME = local.ami_id_ssm_parameter_name INSTANCE_ALLOCATION_STRATEGY = var.config.instance_allocation_strategy INSTANCE_MAX_SPOT_PRICE = var.config.instance_max_spot_price @@ -203,7 +203,9 @@ locals { ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS = jsonencode(var.config.on_demand_failover_for_errors) SCALE_ERRORS = jsonencode(var.config.scale_errors) USE_DEDICATED_HOST = var.config.use_dedicated_host - } + }, var.storage_provider.type == "aws_dynamodb" ? { + EC2_INSTANCE_ARN_PREFIX = local.ec2_instance_arn_prefix + } : {}) scale_down_environment_variables = {} diff --git a/modules/compute-providers/aws/ec2/policies-runner.tf b/modules/compute-providers/aws/ec2/policies-runner.tf index c16077debc..c4f387846b 100644 --- a/modules/compute-providers/aws/ec2/policies-runner.tf +++ b/modules/compute-providers/aws/ec2/policies-runner.tf @@ -4,6 +4,7 @@ data "aws_caller_identity" "current" {} locals { ssm_parameter_arn_prefix = "arn:${var.aws_partition}:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter" + ec2_instance_arn_prefix = "arn:${var.aws_partition}:ec2:${var.aws_region}:${data.aws_caller_identity.current.account_id}:instance/" ssm_config_arn = "${local.ssm_parameter_arn_prefix}${var.ssm.paths.root}/${var.ssm.paths.config}" cloudwatch_config_arn = "${local.ssm_config_arn}/cloudwatch_agent_config_runner" } @@ -167,10 +168,6 @@ data "aws_iam_policy_document" "cloudwatch" { locals { runner_inline_policies = merge( { - ssm_parameters = { - name = "runner-ssm-parameters" - policy_json = data.aws_iam_policy_document.ssm_parameters.json - } describe_tags = { name = "runner-describe-tags" policy_json = data.aws_iam_policy_document.describe_tags.json @@ -184,6 +181,17 @@ locals { policy_json = data.aws_iam_policy_document.terminate_self.json } }, + var.storage_provider.type == "aws_ssm" ? { + ssm_parameters = { + name = "runner-ssm-parameters" + policy_json = data.aws_iam_policy_document.ssm_parameters.json + } + } : { + runner_config_storage = { + name = "runner-config-storage" + policy_json = var.storage_provider.runner.iam_policy_json + } + }, var.config.ssm_enabled ? { session_manager = { name = "runner-ssm-session" diff --git a/modules/compute-providers/aws/ec2/runner-config.tf b/modules/compute-providers/aws/ec2/runner-config.tf index f1d859581c..f562047a7f 100644 --- a/modules/compute-providers/aws/ec2/runner-config.tf +++ b/modules/compute-providers/aws/ec2/runner-config.tf @@ -1,4 +1,5 @@ resource "aws_ssm_parameter" "runner_config_run_as" { + count = var.storage_provider.type == "aws_ssm" ? 1 : 0 name = "${var.ssm.paths.root}/${var.ssm.paths.config}/run_as" type = "String" value = var.runner.run_as_root ? "root" : var.runner.run_as @@ -6,8 +7,19 @@ resource "aws_ssm_parameter" "runner_config_run_as" { } resource "aws_ssm_parameter" "runner_enable_cloudwatch" { + count = var.storage_provider.type == "aws_ssm" ? 1 : 0 name = "${var.ssm.paths.root}/${var.ssm.paths.config}/enable_cloudwatch" type = "String" value = var.config.cloudwatch_agent.enabled tags = local.ssm_parameter_tags } + +moved { + from = aws_ssm_parameter.runner_config_run_as + to = aws_ssm_parameter.runner_config_run_as[0] +} + +moved { + from = aws_ssm_parameter.runner_enable_cloudwatch + to = aws_ssm_parameter.runner_enable_cloudwatch[0] +} diff --git a/modules/compute-providers/aws/ec2/runner-instances.tf b/modules/compute-providers/aws/ec2/runner-instances.tf index f33ab8f532..9734632c47 100644 --- a/modules/compute-providers/aws/ec2/runner-instances.tf +++ b/modules/compute-providers/aws/ec2/runner-instances.tf @@ -90,7 +90,12 @@ locals { hook_job_started = var.runner.hooks.job_started hook_job_completed = var.runner.hooks.job_completed start_runner = templatefile(local.userdata_start_runner[var.runner.os], { - metadata_tags = var.config.metadata_options != null ? var.config.metadata_options.instance_metadata_tags : "enabled" + metadata_tags = var.config.metadata_options != null ? var.config.metadata_options.instance_metadata_tags : "enabled" + storage_provider_type = var.storage_provider.type + dynamodb_config_table_name_base64 = var.storage_provider.type == "aws_dynamodb" ? base64encode(var.storage_provider.runner.config_table_name) : "" + dynamodb_scope_base64 = var.storage_provider.type == "aws_dynamodb" ? base64encode(var.storage_provider.runner.scope) : "" + ec2_instance_arn_prefix_base64 = var.storage_provider.type == "aws_dynamodb" ? base64encode(local.ec2_instance_arn_prefix) : "" + enable_cloudwatch_agent = var.config.cloudwatch_agent.enabled }) ghes_url = var.github.enterprise_server.url ghes_ssl_verify = var.github.enterprise_server.ssl_verify diff --git a/modules/compute-providers/aws/ec2/templates/start-runner-osx.sh b/modules/compute-providers/aws/ec2/templates/start-runner-osx.sh index a6da66116d..c0a274bb45 100644 --- a/modules/compute-providers/aws/ec2/templates/start-runner-osx.sh +++ b/modules/compute-providers/aws/ec2/templates/start-runner-osx.sh @@ -88,6 +88,37 @@ echo "Retrieved ghr:environment tag - ($environment)" echo "Retrieved ghr:ssm_config_path tag - ($ssm_config_path)" echo "Retrieved ghr:runner_name_prefix tag - ($runner_name_prefix)" +%{ if storage_provider_type == "aws_dynamodb" } +dynamodb_config_table_name=$(printf '%s' "${dynamodb_config_table_name_base64}" | openssl base64 -d -A) +dynamodb_scope=$(printf '%s' "${dynamodb_scope_base64}" | openssl base64 -d -A) +ec2_instance_arn_prefix=$(printf '%s' "${ec2_instance_arn_prefix_base64}" | openssl base64 -d -A) + +echo "Retrieving runner bootstrap configuration from DynamoDB" +runner_config_key=$(jq -cn --arg scope "$dynamodb_scope" '{scope:{S:$scope},id:{S:"runner-config"}}') +runner_config_record=$(aws dynamodb get-item \ + --table-name "$dynamodb_config_table_name" \ + --key "$runner_config_key" \ + --consistent-read \ + --projection-expression "#value" \ + --expression-attribute-names '{"#value":"value"}' \ + --region "$region") +runner_config=$(printf '%s' "$runner_config_record" | jq -er '.Item.value.S | fromjson') +unset runner_config_record + +run_as=$(printf '%s' "$runner_config" | jq -r '.run_as') +agent_mode=$(printf '%s' "$runner_config" | jq -r '.agent_mode') +disable_default_labels=$(printf '%s' "$runner_config" | jq -r '.disable_default_labels') +enable_jit_config=$(printf '%s' "$runner_config" | jq -r '.enable_jit_config') +dynamodb_runner_state_table_name=$(printf '%s' "$runner_config" | jq -er '.runner_config_storage.table_name') +dynamodb_access_scope_type=$(printf '%s' "$runner_config" | jq -er '.runner_config_storage.access_scope') +dynamodb_config_id=$(printf '%s' "$runner_config" | jq -er '.runner_config_storage.id') +if [[ "$dynamodb_access_scope_type" != "compute-resource" ]]; then + echo "Unsupported runner configuration access scope" + exit 1 +fi +dynamodb_access_scope="$${ec2_instance_arn_prefix}$instance_id" +unset runner_config +%{ else } parameters=$(aws ssm get-parameters-by-path \ --path "$ssm_config_path" \ --region "$region" \ @@ -108,7 +139,38 @@ echo "Retrieved /$ssm_config_path/enable_jit_config parameter - ($enable_jit_con token_path=$(echo "$parameters" | jq -r '.[] | select(.Name == "'$ssm_config_path'/token_path") | .Value') echo "Retrieved /$ssm_config_path/token_path parameter - ($token_path)" +%{ endif } + +%{ if storage_provider_type == "aws_dynamodb" } +echo "Retrieving one-time runner configuration from DynamoDB" +runner_state_key=$(jq -cn --arg scope "$dynamodb_access_scope" --arg id "$dynamodb_config_id" '{scope:{S:$scope},id:{S:$id}}') +config="" +retrycount=0 +while [[ -z "$config" ]]; do + now_epoch=$(date +%s) + expression_values=$(jq -cn --arg now "$now_epoch" '{":now":{N:$now}}') + if config_record=$(aws dynamodb delete-item \ + --table-name "$dynamodb_runner_state_table_name" \ + --key "$runner_state_key" \ + --condition-expression "attribute_exists(#expires_at) AND #expires_at > :now" \ + --expression-attribute-names '{"#expires_at":"expires_at"}' \ + --expression-attribute-values "$expression_values" \ + --return-values ALL_OLD \ + --region "$region" 2>/dev/null); then + config=$(printf '%s' "$config_record" | jq -er '.Attributes.value.S') + unset config_record + break + fi + retrycount=$((retrycount + 1)) + if [[ $retrycount -gt 40 ]]; then + echo "Runner configuration was unavailable or expired" + exit 1 + fi + echo "Waiting for runner configuration to become available in DynamoDB" + sleep 1 +done +%{ else } echo "Get GH Runner config from AWS SSM" config=$(aws ssm get-parameter --name "$token_path"/"$instance_id" --with-decryption --region "$region" | jq -r ".Parameter | .Value") while [[ -z "$config" ]]; do @@ -119,6 +181,7 @@ done echo "Delete GH Runner token from AWS SSM" aws ssm delete-parameter --name "$token_path"/"$instance_id" --region "$region" +%{ endif } if [ -z "$run_as" ]; then echo "No user specified, using default ec2-user account" diff --git a/modules/compute-providers/aws/ec2/templates/start-runner.ps1 b/modules/compute-providers/aws/ec2/templates/start-runner.ps1 index ae2eeff3c9..8d88b37e8b 100644 --- a/modules/compute-providers/aws/ec2/templates/start-runner.ps1 +++ b/modules/compute-providers/aws/ec2/templates/start-runner.ps1 @@ -77,6 +77,43 @@ Write-Host "Retrieved ghr:runner_name_prefix tag - ($runner_name_prefix)" $ssm_config_path=$tags.Tags.where( {$_.Key -eq 'ghr:ssm_config_path'}).value Write-Host "Retrieved ghr:ssm_config_path tag - ($ssm_config_path)" +%{ if storage_provider_type == "aws_dynamodb" } +$DynamoDbConfigTableName = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String("${dynamodb_config_table_name_base64}")) +$DynamoDbScope = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String("${dynamodb_scope_base64}")) +$Ec2InstanceArnPrefix = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String("${ec2_instance_arn_prefix_base64}")) + +Write-Host "Retrieving runner bootstrap configuration from DynamoDB" +$RunnerConfigKey = @{ + scope = @{ S = $DynamoDbScope } + id = @{ S = "runner-config" } +} | ConvertTo-Json -Compress +$RunnerConfigRecord = aws dynamodb get-item ` + --table-name $DynamoDbConfigTableName ` + --key $RunnerConfigKey ` + --consistent-read ` + --projection-expression "#value" ` + --expression-attribute-names '{"#value":"value"}' ` + --region $Region | ConvertFrom-Json +if ($LASTEXITCODE -ne 0 -or -not $RunnerConfigRecord.Item.value.S) { + throw "Runner bootstrap configuration is unavailable" +} +$RunnerConfig = $RunnerConfigRecord.Item.value.S | ConvertFrom-Json +$RunnerConfigRecord = $null + +$run_as = $RunnerConfig.run_as +$agent_mode = $RunnerConfig.agent_mode +$disable_default_labels = $RunnerConfig.disable_default_labels.ToString().ToLowerInvariant() +$enable_jit_config = $RunnerConfig.enable_jit_config.ToString().ToLowerInvariant() +$enable_cloudwatch_agent = "${enable_cloudwatch_agent}" +$DynamoDbRunnerStateTableName = $RunnerConfig.runner_config_storage.table_name +$DynamoDbAccessScopeType = $RunnerConfig.runner_config_storage.access_scope +$DynamoDbConfigId = $RunnerConfig.runner_config_storage.id +if ($DynamoDbAccessScopeType -ne "compute-resource") { + throw "Unsupported runner configuration access scope" +} +$DynamoDbAccessScope = "$Ec2InstanceArnPrefix$InstanceId" +$RunnerConfig = $null +%{ else } $parameters=$(aws ssm get-parameters-by-path --path "$ssm_config_path" --region "$Region" --query "Parameters[*].{Name:Name,Value:Value}") | ConvertFrom-Json Write-Host "Retrieved parameters from AWS SSM" @@ -97,6 +134,7 @@ Write-Host "Retrieved $ssm_config_path/enable_jit_config parameter - ($enable_j $token_path=$parameters.where( {$_.Name -eq "$ssm_config_path/token_path"}).value Write-Host "Retrieved $ssm_config_path/token_path parameter - ($token_path)" +%{ endif } if ($enable_cloudwatch_agent -eq "true") @@ -107,6 +145,40 @@ if ($enable_cloudwatch_agent -eq "true") ## Configure the runner +%{ if storage_provider_type == "aws_dynamodb" } +Write-Host "Retrieving one-time runner configuration from DynamoDB" +$RunnerStateKey = @{ + scope = @{ S = $DynamoDbAccessScope } + id = @{ S = $DynamoDbConfigId } +} | ConvertTo-Json -Compress +$config = $null +$i = 0 +do { + $NowEpoch = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds().ToString() + $ExpressionValues = @{ ":now" = @{ N = $NowEpoch } } | ConvertTo-Json -Compress + $ConfigRecordRaw = aws dynamodb delete-item ` + --table-name $DynamoDbRunnerStateTableName ` + --key $RunnerStateKey ` + --condition-expression "attribute_exists(#expires_at) AND #expires_at > :now" ` + --expression-attribute-names '{"#expires_at":"expires_at"}' ` + --expression-attribute-values $ExpressionValues ` + --return-values ALL_OLD ` + --region $Region 2>$null + if ($LASTEXITCODE -eq 0) { + $config = ($ConfigRecordRaw | ConvertFrom-Json).Attributes.value.S + $ConfigRecordRaw = $null + break + } + + Write-Host "Waiting for runner configuration to become available in DynamoDB ($i/40)" + Start-Sleep 1 + $i++ +} while (($null -eq $config) -and ($i -lt 40)) + +if ($null -eq $config) { + throw "Runner configuration was unavailable or expired" +} +%{ else } Write-Host "Get GH Runner config from AWS SSM" $config = $null $i = 0 @@ -119,6 +191,7 @@ do { Write-Host "Delete GH Runner token from AWS SSM" aws ssm delete-parameter --name "$token_path/$InstanceId" --region $Region +%{ endif } # Create or update user if (-not($run_as)) { diff --git a/modules/compute-providers/aws/ec2/templates/start-runner.sh b/modules/compute-providers/aws/ec2/templates/start-runner.sh index 7f2c0f82c5..b6533df1dc 100644 --- a/modules/compute-providers/aws/ec2/templates/start-runner.sh +++ b/modules/compute-providers/aws/ec2/templates/start-runner.sh @@ -159,6 +159,38 @@ echo "Retrieved ghr:environment tag - ($environment)" echo "Retrieved ghr:ssm_config_path tag - ($ssm_config_path)" echo "Retrieved ghr:runner_name_prefix tag - ($runner_name_prefix)" +%{ if storage_provider_type == "aws_dynamodb" } +dynamodb_config_table_name=$(printf '%s' "${dynamodb_config_table_name_base64}" | openssl base64 -d -A) +dynamodb_scope=$(printf '%s' "${dynamodb_scope_base64}" | openssl base64 -d -A) +ec2_instance_arn_prefix=$(printf '%s' "${ec2_instance_arn_prefix_base64}" | openssl base64 -d -A) + +echo "Retrieving runner bootstrap configuration from DynamoDB" +runner_config_key=$(jq -cn --arg scope "$dynamodb_scope" '{scope:{S:$scope},id:{S:"runner-config"}}') +runner_config_record=$(aws dynamodb get-item \ + --table-name "$dynamodb_config_table_name" \ + --key "$runner_config_key" \ + --consistent-read \ + --projection-expression "#value" \ + --expression-attribute-names '{"#value":"value"}' \ + --region "$region") +runner_config=$(printf '%s' "$runner_config_record" | jq -er '.Item.value.S | fromjson') +unset runner_config_record + +run_as=$(printf '%s' "$runner_config" | jq -r '.run_as') +agent_mode=$(printf '%s' "$runner_config" | jq -r '.agent_mode') +disable_default_labels=$(printf '%s' "$runner_config" | jq -r '.disable_default_labels') +enable_jit_config=$(printf '%s' "$runner_config" | jq -r '.enable_jit_config') +enable_cloudwatch_agent="${enable_cloudwatch_agent}" +dynamodb_runner_state_table_name=$(printf '%s' "$runner_config" | jq -er '.runner_config_storage.table_name') +dynamodb_access_scope_type=$(printf '%s' "$runner_config" | jq -er '.runner_config_storage.access_scope') +dynamodb_config_id=$(printf '%s' "$runner_config" | jq -er '.runner_config_storage.id') +if [[ "$dynamodb_access_scope_type" != "compute-resource" ]]; then + echo "Unsupported runner configuration access scope" + exit 1 +fi +dynamodb_access_scope="$${ec2_instance_arn_prefix}$instance_id" +unset runner_config +%{ else } parameters=$(aws ssm get-parameters-by-path --path "$ssm_config_path" --region "$region" --query "Parameters[*].{Name:Name,Value:Value}") echo "Retrieved parameters from AWS SSM ($parameters)" @@ -179,6 +211,7 @@ echo "Retrieved /$ssm_config_path/enable_jit_config parameter - ($enable_jit_con token_path=$(echo "$parameters" | jq --arg ssm_config_path "$ssm_config_path" -r '.[] | select(.Name == "'$ssm_config_path'/token_path") | .Value') echo "Retrieved /$ssm_config_path/token_path parameter - ($token_path)" +%{ endif } if [[ "$xray_trace_id" != "" ]]; then # run xray service @@ -199,6 +232,36 @@ fi ## Configure the runner +%{ if storage_provider_type == "aws_dynamodb" } +echo "Retrieving one-time runner configuration from DynamoDB" +runner_state_key=$(jq -cn --arg scope "$dynamodb_access_scope" --arg id "$dynamodb_config_id" '{scope:{S:$scope},id:{S:$id}}') +config="" +retrycount=0 +while [[ -z "$config" ]]; do + now_epoch=$(date +%s) + expression_values=$(jq -cn --arg now "$now_epoch" '{":now":{N:$now}}') + if config_record=$(aws dynamodb delete-item \ + --table-name "$dynamodb_runner_state_table_name" \ + --key "$runner_state_key" \ + --condition-expression "attribute_exists(#expires_at) AND #expires_at > :now" \ + --expression-attribute-names '{"#expires_at":"expires_at"}' \ + --expression-attribute-values "$expression_values" \ + --return-values ALL_OLD \ + --region "$region" 2>/dev/null); then + config=$(printf '%s' "$config_record" | jq -er '.Attributes.value.S') + unset config_record + break + fi + + retrycount=$((retrycount + 1)) + if [[ $retrycount -gt 40 ]]; then + echo "Runner configuration was unavailable or expired" + exit 1 + fi + echo "Waiting for runner configuration to become available in DynamoDB" + sleep 1 +done +%{ else } echo "Get GH Runner config from AWS SSM" config=$(aws ssm get-parameter --name "$token_path"/"$instance_id" --with-decryption --region "$region" | jq -r ".Parameter | .Value") while [[ -z "$config" ]]; do @@ -209,6 +272,7 @@ done echo "Delete GH Runner token from AWS SSM" aws ssm delete-parameter --name "$token_path"/"$instance_id" --region "$region" +%{ endif } if [ -z "$run_as" ]; then echo "No user specified, using default ec2-user account" diff --git a/modules/compute-providers/aws/ec2/tests/provider.tftest.hcl b/modules/compute-providers/aws/ec2/tests/provider.tftest.hcl index bc92537279..5de6ef2f1c 100644 --- a/modules/compute-providers/aws/ec2/tests/provider.tftest.hcl +++ b/modules/compute-providers/aws/ec2/tests/provider.tftest.hcl @@ -367,11 +367,11 @@ run "separates_provider_runner_and_ssm_tags" { assert { condition = ( - aws_ssm_parameter.runner_config_run_as.tags["Name"] == "ssm-name" - && aws_ssm_parameter.runner_config_run_as.tags["Scope"] == "ssm" - && aws_ssm_parameter.runner_config_run_as.tags["SsmOnly"] == "ssm" - && !contains(keys(aws_ssm_parameter.runner_config_run_as.tags), "RunnerOnly") - && !contains(keys(aws_ssm_parameter.runner_config_run_as.tags), "ghr:environment") + aws_ssm_parameter.runner_config_run_as[0].tags["Name"] == "ssm-name" + && aws_ssm_parameter.runner_config_run_as[0].tags["Scope"] == "ssm" + && aws_ssm_parameter.runner_config_run_as[0].tags["SsmOnly"] == "ssm" + && !contains(keys(aws_ssm_parameter.runner_config_run_as[0].tags), "RunnerOnly") + && !contains(keys(aws_ssm_parameter.runner_config_run_as[0].tags), "ghr:environment") ) error_message = "EC2 SSM parameters must merge SSM component tags over provider tags." } @@ -449,3 +449,77 @@ run "requires_distribution_object_when_sync_is_enabled" { expect_failures = [terraform_data.validate_config] } + +run "dynamodb_bootstrap_is_opt_in_and_compute_scoped" { + command = plan + + variables { + config = { + vpc_id = "vpc-12345678" + subnet_ids = ["subnet-12345678"] + instance_types = ["m5.large"] + ami = { + filter = { state = ["available"] } + owners = ["amazon"] + id_ssm_parameter = { + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/ami-id" + } + kms_key = null + } + binaries_syncer = { + enabled = false + } + cloudwatch_agent = { + enabled = false + } + managed_security_group_enabled = true + } + + runner = { + os = "windows" + architecture = "x64" + iam = { + role = { + arn = "arn:aws:iam::123456789012:role/provider-test-runner" + name = "provider-test-runner" + } + } + } + + storage_provider = { + type = "aws_dynamodb" + runner = { + config_table_name = "provider-test-config" + runner_state_table_name = "provider-test-runner-state" + scope = "entry#unsafe-$(value)#bootstrap" + iam_policy_json = jsonencode({ Version = "2012-10-17", Statement = [] }) + } + } + } + + assert { + condition = ( + length(aws_ssm_parameter.runner_config_run_as) == 0 + && length(aws_ssm_parameter.runner_enable_cloudwatch) == 0 + && contains(keys(output.provider.policies.runner.inline_policies), "runner_config_storage") + && !contains(keys(output.provider.policies.runner.inline_policies), "ssm_parameters") + && output.provider.environment_variables.scale_up["EC2_INSTANCE_ARN_PREFIX"] == "arn:aws:ec2:eu-west-1:123456789012:instance/" + && output.provider.environment_variables.pool["EC2_INSTANCE_ARN_PREFIX"] == "arn:aws:ec2:eu-west-1:123456789012:instance/" + ) + error_message = "DynamoDB-selected EC2 runners must replace SSM bootstrap resources and expose the matching compute-resource ARN prefix." + } + + assert { + condition = ( + strcontains(base64decode(aws_launch_template.runner.user_data), base64encode("provider-test-config")) + && strcontains(base64decode(aws_launch_template.runner.user_data), base64encode("entry#unsafe-$(value)#bootstrap")) + && strcontains(base64decode(aws_launch_template.runner.user_data), base64encode("arn:aws:ec2:eu-west-1:123456789012:instance/")) + && !strcontains(base64decode(aws_launch_template.runner.user_data), "entry#unsafe-$(value)#bootstrap") + && strcontains(base64decode(aws_launch_template.runner.user_data), "aws dynamodb delete-item") + && strcontains(base64decode(aws_launch_template.runner.user_data), "attribute_exists(#expires_at) AND #expires_at > :now") + && strcontains(base64decode(aws_launch_template.runner.user_data), "--return-values ALL_OLD") + && !strcontains(base64decode(aws_launch_template.runner.user_data), "aws ssm get-parameters --names") + ) + error_message = "DynamoDB bootstrap must base64-embed user-controlled locators and atomically consume only an unexpired per-instance record." + } +} diff --git a/modules/compute-providers/aws/ec2/variables.tf b/modules/compute-providers/aws/ec2/variables.tf index da04a3b37c..076c019876 100644 --- a/modules/compute-providers/aws/ec2/variables.tf +++ b/modules/compute-providers/aws/ec2/variables.tf @@ -345,6 +345,43 @@ variable "ssm" { nullable = false } +variable "storage_provider" { + description = "Runner-side storage locator and opaque IAM policy supplied by runner-config. The default preserves the existing SSM bootstrap path." + type = object({ + type = string + runner = object({ + config_table_name = optional(string, null) + runner_state_table_name = optional(string, null) + scope = optional(string, null) + iam_policy_json = optional(string, null) + }) + }) + default = { + type = "aws_ssm" + runner = { + config_table_name = null + runner_state_table_name = null + scope = null + iam_policy_json = null + } + } + + validation { + condition = contains(["aws_ssm", "aws_dynamodb"], var.storage_provider.type) + error_message = "storage_provider.type must be aws_ssm or aws_dynamodb." + } + + validation { + condition = var.storage_provider.type != "aws_dynamodb" || ( + var.storage_provider.runner.config_table_name != null && + var.storage_provider.runner.runner_state_table_name != null && + var.storage_provider.runner.scope != null && + var.storage_provider.runner.iam_policy_json != null + ) + error_message = "aws_dynamodb storage requires non-null runner config table, runner-state table, bootstrap scope, and IAM policy capabilities." + } +} + variable "observability" { description = <<-EOT CloudWatch Logs settings available to compute-provider runner log groups. diff --git a/modules/multi-runner/README.md b/modules/multi-runner/README.md index 61a558389f..bbb95b67b5 100644 --- a/modules/multi-runner/README.md +++ b/modules/multi-runner/README.md @@ -157,6 +157,7 @@ multi_runner_config = { | [runner\_binaries](#module\_runner\_binaries) | ../runner-binaries-syncer | n/a | | [runners](#module\_runners) | ../runners | n/a | | [ssm](#module\_ssm) | ../ssm | n/a | +| [storage\_aws\_dynamodb](#module\_storage\_aws\_dynamodb) | ../storage-providers/aws/dynamodb | n/a | | [webhook](#module\_webhook) | ../webhook | n/a | ## Resources @@ -168,6 +169,7 @@ multi_runner_config = { | [aws_sqs_queue_policy.build_queue_dlq_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource | | [aws_sqs_queue_policy.build_queue_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue_policy) | resource | | [random_string.random](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | +| [aws_caller_identity.storage](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_iam_policy_document.deny_insecure_transport](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | ## Inputs @@ -200,6 +202,7 @@ multi_runner_config = { | [global\_config\_observability](#input\_global\_config\_observability) | Global observability configuration shared by all runner lanes.

global\_config\_observability = {
logs.level: "Log level for module resources."
logs.retention\_in\_days: "CloudWatch log retention period in days."
logs.kms\_key\_id: "KMS key ID used to encrypt CloudWatch log groups."
logs.class: "CloudWatch log group class."
logs.tags: "Tags applied to CloudWatch log groups."
tracing.mode: "Tracing mode used by instrumented resources."
tracing.capture\_http\_requests: "Whether HTTP requests are captured by tracing."
tracing.capture\_error: "Whether errors are captured by tracing."
metrics.enabled: "Whether module metrics are enabled."
metrics.namespace: "CloudWatch namespace used for module metrics."
metrics.metric.github\_app\_rate\_limit.enabled: "Whether GitHub App rate-limit metrics are emitted."
metrics.metric.job\_retry.enabled: "Whether job-retry metrics are emitted."
metrics.metric.spot\_termination\_warning.enabled: "Whether spot-termination warning metrics are emitted."
} | 
object({
    logs = optional(object({
      level             = optional(string, "info")
      retention_in_days = optional(number, 180)
      kms_key_id        = optional(string, null)
      class             = optional(string, "STANDARD")
      tags              = optional(map(string), {})
    }), {})
    tracing = optional(object({
      mode                  = optional(string, null)
      capture_http_requests = optional(bool, false)
      capture_error         = optional(bool, false)
    }), {})
    metrics = optional(object({
      enabled   = optional(bool, false)
      namespace = optional(string, "GitHub Runners")
      metric = optional(object({
        github_app_rate_limit = optional(object({
          enabled = optional(bool, true)
        }), {})
        job_retry = optional(object({
          enabled = optional(bool, true)
        }), {})
        spot_termination_warning = optional(object({
          enabled = optional(bool, true)
        }), {})
      }), {})
    }), {})
  })
| `{}` | no | | [global\_config\_orchestration\_provider](#input\_global\_config\_orchestration\_provider) | Global orchestration-provider configuration shared by all runner lanes.

global\_config\_orchestration\_provider = {
webhook: {
queue\_selection\_strategy: "Strategy used to select the build queue for a webhook event."
eventbridge.enabled: "Whether EventBridge integration is enabled for webhook events."
eventbridge.accept\_events: "Event types accepted by the EventBridge integration."
matcher\_config\_parameter\_store\_tier: "SSM Parameter Store tier used for matcher configuration."
runner.boot\_time\_in\_minutes: "Expected runner boot time used by orchestration."
runner.ephemeral: "Whether runners created by the orchestration provider are ephemeral."
runner.jit\_config\_enabled: "Whether JIT runner configuration is enabled."
runner.maximum\_count: "Maximum number of runners that orchestration may create."
github.repository\_white\_list: "Repositories allowed to use the webhook configuration."
lambda.artifact.zip: "Local ZIP artifact used for orchestration Lambda functions."
lambda.artifact.s3.key: "S3 object key for the orchestration Lambda artifact."
lambda.artifact.s3.object\_version: "Optional S3 object version for the orchestration Lambda artifact."
lambda.scale.up.memory\_size: "Memory allocated to the scale-up Lambda."
lambda.scale.up.timeout: "Timeout in seconds for the scale-up Lambda."
lambda.scale.up.reserved\_concurrent\_executions: "Reserved concurrent executions for the scale-up Lambda."
lambda.scale.up.job\_queued\_check\_enabled: "Whether the scale-up Lambda checks queued jobs."
lambda.scale.up.event\_source\_mapping.batch\_size: "Maximum records passed to one scale-up Lambda invocation."
lambda.scale.up.event\_source\_mapping.maximum\_batching\_window\_in\_seconds: "Maximum time to batch records before invoking the scale-up Lambda."
lambda.scale.up.tags: "Tags applied to the scale-up Lambda."
lambda.scale.down.memory\_size: "Memory allocated to the scale-down Lambda."
lambda.scale.down.timeout: "Timeout in seconds for the scale-down Lambda."
lambda.scale.down.schedule\_expression: "Schedule expression for scale-down processing."
lambda.scale.down.minimum\_running\_time\_in\_minutes: "Minimum runner lifetime before scale-down."
lambda.scale.down.idle\_config: "Scheduled minimum idle-runner pool settings."
lambda.scale.down.idle\_config.cron: "Cron expression defining when the idle-runner count applies."
lambda.scale.down.idle\_config.timeZone: "Time zone used to evaluate the idle-runner schedule."
lambda.scale.down.idle\_config.idleCount: "Minimum number of idle runners maintained during the schedule."
lambda.scale.down.idle\_config.evictionStrategy: "Strategy used when evicting idle runners."
lambda.scale.down.tags: "Tags applied to the scale-down Lambda."
lambda.webhook.artifact.zip: "Local ZIP artifact used for the webhook Lambda."
lambda.webhook.artifact.s3.key: "S3 object key for the webhook Lambda artifact."
lambda.webhook.artifact.s3.object\_version: "Optional S3 object version for the webhook Lambda artifact."
lambda.webhook.api\_gateway\_access\_log\_settings: "API Gateway access-log destination and format."
lambda.webhook.api\_gateway\_access\_log\_settings.destination\_arn: "ARN of the API Gateway access-log destination."
lambda.webhook.api\_gateway\_access\_log\_settings.format: "API Gateway access-log format."
lambda.webhook.memory\_size: "Memory allocated to the webhook Lambda."
lambda.webhook.timeout: "Timeout in seconds for the webhook Lambda."
lambda.webhook.tags: "Tags applied to the webhook Lambda."
lambda.pool.memory\_size: "Memory allocated to the pool Lambda."
lambda.pool.timeout: "Timeout in seconds for the pool Lambda."
lambda.pool.reserved\_concurrent\_executions: "Reserved concurrent executions for the pool Lambda."
lambda.pool.config: "Scheduled runner-pool size configuration."
lambda.pool.config.schedule\_expression: "Schedule expression for the pool size."
lambda.pool.config.schedule\_expression\_timezone: "Time zone used to evaluate the pool schedule."
lambda.pool.config.size: "Runner pool size applied by the schedule."
lambda.pool.include\_busy\_runners: "Whether busy runners are included in pool sizing."
lambda.pool.runner\_owner: "GitHub organization that owns the runner pool."
lambda.pool.tags: "Tags applied to the pool Lambda."
queue.delay\_webhook\_event: "Seconds a webhook event remains invisible in the build queue before processing."
queue.job\_queue\_retention\_in\_seconds: "Seconds a queued job is retained before it is purged."
queue.visibility\_timeout\_seconds: "Build queue visibility timeout in seconds."
queue.redrive\_build\_queue.enabled: "Whether the build queue dead-letter queue is enabled."
queue.redrive\_build\_queue.maxReceiveCount: "Maximum receives before a message is moved to the dead-letter queue."
queue.tags: "Tags applied to build queues."
queue.encryption.kms\_data\_key\_reuse\_period\_seconds: "KMS data-key reuse period for queue encryption."
queue.encryption.kms\_master\_key\_id: "KMS key ID used for queue encryption."
queue.encryption.sqs\_managed\_sse\_enabled: "Whether SQS-managed server-side encryption is enabled."
}
} | 
object({
    webhook = optional(object({
      queue_selection_strategy = optional(string, "first")
      eventbridge = optional(object({
        enabled       = optional(bool, true)
        accept_events = optional(list(string), [])
      }), {})
      matcher_config_parameter_store_tier = optional(string, "Standard")
      runner = optional(object({
        boot_time_in_minutes = optional(number, 5)
        ephemeral            = optional(bool, false)
        jit_config_enabled   = optional(bool, null)
        maximum_count        = optional(number, null)
      }), {})

      github = optional(object({
        repository_white_list = optional(list(string), [])
      }), {})

      lambda = optional(object({
        artifact = optional(object({
          zip = optional(string, null)
          s3 = optional(object({
            key            = string
            object_version = optional(string, null)
          }), null)
        }), {})
        scale = optional(object({
          up = optional(object({
            memory_size                    = optional(number, 512)
            timeout                        = optional(number, 30)
            reserved_concurrent_executions = optional(number, 1)
            job_queued_check_enabled       = optional(bool, null)
            event_source_mapping = optional(object({
              batch_size                         = optional(number, 10)
              maximum_batching_window_in_seconds = optional(number, 0)
            }), {})
            tags = optional(map(string), {})
          }), {})
          down = optional(object({
            memory_size                     = optional(number, 512)
            timeout                         = optional(number, 60)
            schedule_expression             = optional(string, "cron(*/5 * * * ? *)")
            minimum_running_time_in_minutes = optional(number, null)
            idle_config = optional(list(object({
              cron             = string
              timeZone         = string
              idleCount        = number
              evictionStrategy = optional(string, "oldest_first")
            })), [])
            tags = optional(map(string), {})
          }), {})
        }), {})
        webhook = optional(object({
          artifact = optional(object({
            zip = optional(string, null)
            s3 = optional(object({
              key            = string
              object_version = optional(string, null)
            }), null)
          }), {})
          api_gateway_access_log_settings = optional(object({
            destination_arn = string
            format          = string
          }), null)
          memory_size = optional(number, 256)
          timeout     = optional(number, 10)
          tags        = optional(map(string), {})
        }), {})
        pool = optional(object({
          memory_size                    = optional(number, 512)
          timeout                        = optional(number, 60)
          reserved_concurrent_executions = optional(number, 1)
          config = optional(list(object({
            schedule_expression          = string
            schedule_expression_timezone = optional(string)
            size                         = number
          })), [])
          include_busy_runners = optional(bool, false)
          runner_owner         = optional(string, null)
          tags                 = optional(map(string), {})
        }), {})
      }), {})

      queue = optional(object({
        delay_webhook_event            = optional(number, 30)
        job_queue_retention_in_seconds = optional(number, 86400)
        visibility_timeout_seconds     = optional(number, 180)
        redrive_build_queue = optional(object({
          enabled         = optional(bool, false)
          maxReceiveCount = optional(number, null)
          }), {
          enabled         = false
          maxReceiveCount = null
        })
        tags = optional(map(string), {})
        encryption = optional(object({
          kms_data_key_reuse_period_seconds = number
          kms_master_key_id                 = string
          sqs_managed_sse_enabled           = bool
          }), {
          kms_data_key_reuse_period_seconds = null
          kms_master_key_id                 = null
          sqs_managed_sse_enabled           = true
        })
      }), {})
    }), {})
  })
| `{}` | no | | [global\_config\_ssm](#input\_global\_config\_ssm) | Global SSM configuration shared by all runner lanes.

global\_config\_ssm = {
paths.root: "Root path for SSM parameters."
paths.app: "Path segment for application parameters."
paths.webhook: "Path segment for webhook parameters."
paths.tokens: "Path segment for runner token parameters."
paths.config: "Path segment for runner configuration parameters."
kms\_key\_id: "KMS key ID used to encrypt SSM parameters."
tags: "Tags applied to SSM resources."
parameters.tags: "Tags applied to runner configuration parameters."
housekeeper.schedule\_expression: "Schedule for the SSM parameter housekeeper."
housekeeper.state: "EventBridge rule state for the SSM parameter housekeeper."
housekeeper.tags: "Tags applied to the SSM housekeeper resources."
housekeeper.lambda.artifact.zip: "Local ZIP artifact used for the SSM housekeeper Lambda."
housekeeper.lambda.artifact.s3.key: "S3 object key for the SSM housekeeper Lambda artifact."
housekeeper.lambda.artifact.s3.object\_version: "Optional S3 object version for the SSM housekeeper artifact."
housekeeper.lambda.memory\_size: "Memory allocated to the SSM housekeeper Lambda."
housekeeper.lambda.timeout: "Timeout in seconds for the SSM housekeeper Lambda."
housekeeper.config.tokenPath: "Parameter path containing runner tokens to clean up."
housekeeper.config.minimumDaysOld: "Minimum age in days before an old token is eligible for cleanup."
housekeeper.config.dryRun: "Whether the SSM housekeeper reports cleanup without deleting parameters."
} | 
object({
    paths = optional(object({
      root    = optional(string, null)
      app     = optional(string, "app")
      webhook = optional(string, "webhook")
      tokens  = optional(string, "runners/tokens")
      config  = optional(string, "runners/config")
    }), {})
    kms_key_id = optional(string, null)
    tags       = optional(map(string), {})
    parameters = optional(object({
      tags = optional(map(string), {})
    }), {})
    housekeeper = optional(object({
      schedule_expression = optional(string, "rate(1 day)")
      state               = optional(string, "ENABLED")
      tags                = optional(map(string), {})
      lambda = optional(object({
        artifact = optional(object({
          zip = optional(string, null)
          s3 = optional(object({
            key            = string
            object_version = optional(string, null)
          }), null)
        }), {})
        memory_size = optional(number, 512)
        timeout     = optional(number, 60)
      }), {})
      config = optional(object({
        tokenPath      = optional(string, null)
        minimumDaysOld = optional(number, 1)
        dryRun         = optional(bool, false)
      }), {})
    }), {})
  })
| `{}` | no | +| [global\_config\_storage\_provider](#input\_global\_config\_storage\_provider) | Global runner-configuration storage provider selection. Omit the DynamoDB block to retain the existing SSM backend. | 
object({
    aws = optional(object({
      dynamodb = optional(object({
        config = optional(object({
          kms_key_arn                    = optional(string, null)
          point_in_time_recovery_enabled = optional(bool, true)
          deletion_protection_enabled    = optional(bool, false)
          tags                           = optional(map(string), {})
        }), {})
        runner_state = optional(object({
          kms_key_arn                    = optional(string, null)
          point_in_time_recovery_enabled = optional(bool, false)
          deletion_protection_enabled    = optional(bool, false)
          runner_config_ttl_seconds      = optional(number, 86400)
          runner_state_ttl_seconds       = optional(number, 604800)
          tags                           = optional(map(string), {})
        }), {})
      }), null)
      ssm = optional(object({}), null)
    }), {})
  })
| `{}` | no | | [iam\_overrides](#input\_iam\_overrides) | This map provides the possibility to override some IAM defaults. The following attributes are supported: `instance_profile_name` overrides the instance profile name used in the launch template. `runner_role_arn` overrides the IAM role ARN used for the runner instances. | 
object({
    override_instance_profile = optional(bool, null)
    instance_profile_name     = optional(string, null)
    override_runner_role      = optional(bool, null)
    runner_role_arn           = optional(string, null)
  })
| 
{
  "instance_profile_name": null,
  "override_instance_profile": false,
  "override_runner_role": false,
  "runner_role_arn": null
}
| no | | [instance\_profile\_path](#input\_instance\_profile\_path) | The path that will be added to the instance\_profile, if not set the environment name will be used. | `string` | `null` | no | | [instance\_termination\_watcher](#input\_instance\_termination\_watcher) | Configuration for the spot termination watcher lambda function. This feature is Beta, changes will not trigger a major release as long in beta.

`enable`: Enable or disable the spot termination watcher.
`enable_runner_deregistration`: Enable or disable deregistering the runner from GitHub when its EC2 instance is terminated.
`environment_variables`: Additional environment variables to merge into the Lambda configuration.
`memory_size`: Memory size limit in MB of the lambda.
`s3_key`: S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas.
`s3_object_version`: S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket.
`timeout`: Time out of the lambda in seconds.
`zip`: File location of the lambda zip file. | 
object({
    enable = optional(bool, false)
    features = optional(object({
      enable_spot_termination_handler              = optional(bool, true)
      enable_spot_termination_notification_watcher = optional(bool, true)
    }), {})
    enable_runner_deregistration = optional(bool, true)
    environment_variables        = optional(map(string), {})
    memory_size                  = optional(number, null)
    s3_key                       = optional(string, null)
    s3_object_version            = optional(string, null)
    timeout                      = optional(number, null)
    zip                          = optional(string, null)
  })
| `{}` | no | diff --git a/modules/multi-runner/config.experimental.resolved.tf b/modules/multi-runner/config.experimental.resolved.tf index b7e2e26ab0..630de0b105 100644 --- a/modules/multi-runner/config.experimental.resolved.tf +++ b/modules/multi-runner/config.experimental.resolved.tf @@ -24,6 +24,7 @@ locals { runner = var.global_config.runner github = var.global_config_github lambda = var.global_config_lambda + storage_provider = var.global_config_storage_provider orchestration_provider = var.global_config_orchestration_provider ssm = var.global_config_ssm observability = var.global_config_observability @@ -37,6 +38,7 @@ locals { runner = local.stable_to_v2_runner github = local.stable_to_v2_github lambda = local.stable_to_v2_lambda + storage_provider = local.stable_to_v2_storage_provider orchestration_provider = local.stable_to_v2_orchestration_provider ssm = local.stable_to_v2_ssm observability = local.stable_to_v2_observability diff --git a/modules/multi-runner/config.experimental.translation.tf b/modules/multi-runner/config.experimental.translation.tf index 07df28e2e7..d1371e4cee 100644 --- a/modules/multi-runner/config.experimental.translation.tf +++ b/modules/multi-runner/config.experimental.translation.tf @@ -59,6 +59,13 @@ locals { } } + stable_to_v2_storage_provider = { + aws = { + dynamodb = null + ssm = {} + } + } + stable_to_v2_orchestration_provider = { webhook = { queue_selection_strategy = var.queue_selection_strategy diff --git a/modules/multi-runner/storage-provider.tf b/modules/multi-runner/storage-provider.tf new file mode 100644 index 0000000000..c680a5a305 --- /dev/null +++ b/modules/multi-runner/storage-provider.tf @@ -0,0 +1,160 @@ +locals { + requested_storage_provider_types = compact([ + try(local.normalized_config.storage_provider.aws.dynamodb, null) != null ? "aws_dynamodb" : "", + try(local.normalized_config.storage_provider.aws.ssm, null) != null ? "aws_ssm" : "", + ]) + + storage_provider_type = local.use_v2_config ? ( + length(local.requested_storage_provider_types) == 0 + ? "aws_ssm" + : one(local.requested_storage_provider_types) + ) : "aws_ssm" + + default_dynamodb_storage_provider = { + config = { + kms_key_arn = null + point_in_time_recovery_enabled = true + deletion_protection_enabled = false + tags = {} + } + runner_state = { + kms_key_arn = null + point_in_time_recovery_enabled = false + deletion_protection_enabled = false + runner_config_ttl_seconds = 86400 + runner_state_ttl_seconds = 604800 + tags = {} + } + } + + dynamodb_storage_provider = coalesce( + try(local.normalized_config.storage_provider.aws.dynamodb, null), + local.default_dynamodb_storage_provider, + ) + + storage_runner_matcher_config_by_key = { + for k, v in local.runner_matcher_config : format("%03d-%s", v.matcherConfig.priority, k) => merge(v, { + key = k + computeProvider = lower(trimspace(v.computeProvider)) + }) + } + storage_runner_matcher_config = [ + for k in sort(keys(local.storage_runner_matcher_config_by_key)) : local.storage_runner_matcher_config_by_key[k] + ] + + dynamodb_global_records = local.storage_provider_type == "aws_dynamodb" ? { + github_app_credentials = sensitive(jsonencode(concat( + [{ + appId = try(tonumber(local.normalized_config.github.app.id), 0) + privateKeyBase64 = local.normalized_config.github.app.key_base64 + }], + [for app in local.normalized_config.github.additional_apps : merge( + { + appId = try(tonumber(app.id), 0) + privateKeyBase64 = app.key_base64 + }, + app.installation_id == null ? {} : { + installationId = try(tonumber(app.installation_id), 0) + }, + )], + ))) + github_webhook_secret = sensitive(local.normalized_config.github.app.webhook_secret) + runner_matcher_config = jsonencode(local.storage_runner_matcher_config) + } : { + github_app_credentials = sensitive("") + github_webhook_secret = sensitive("") + runner_matcher_config = "" + } + + dynamodb_entry_records = { + for entry_id, entry in local.effective_config.multi_runner_config : entry_id => { + run_as = entry.runner.run_as_root ? "root" : entry.runner.run_as + agent_mode = entry.orchestration_provider.webhook.runner.ephemeral ? "ephemeral" : "persistent" + disable_default_labels = entry.runner.disable_default_labels + enable_jit_config = entry.orchestration_provider.webhook.runner.jit_config_enabled + } + if local.storage_provider_type == "aws_dynamodb" + } +} + +data "aws_caller_identity" "storage" { + count = local.storage_provider_type == "aws_dynamodb" ? 1 : 0 +} + +module "storage_aws_dynamodb" { + source = "../storage-providers/aws/dynamodb" + count = local.storage_provider_type == "aws_dynamodb" ? 1 : 0 + + prefix = var.prefix + tags = merge( + local.effective_config.tags, + { "ghr:environment" = var.prefix }, + ) + config = { + config = local.dynamodb_storage_provider.config + runner_state = { + kms_key_arn = local.dynamodb_storage_provider.runner_state.kms_key_arn + point_in_time_recovery_enabled = local.dynamodb_storage_provider.runner_state.point_in_time_recovery_enabled + deletion_protection_enabled = local.dynamodb_storage_provider.runner_state.deletion_protection_enabled + tags = local.dynamodb_storage_provider.runner_state.tags + } + } + entry_ids = keys(local.effective_config.multi_runner_config) + runner_config_access_scope_prefixes = { + for entry_id in keys(local.effective_config.multi_runner_config) : + entry_id => "arn:${var.aws_partition}:ec2:${var.aws_region}:${data.aws_caller_identity.storage[0].account_id}:instance/" + } + runner_config_ttl_seconds = local.dynamodb_storage_provider.runner_state.runner_config_ttl_seconds + runner_state_ttl_seconds = local.dynamodb_storage_provider.runner_state.runner_state_ttl_seconds + global_records = local.dynamodb_global_records + entry_records = local.dynamodb_entry_records +} + +locals { + dynamodb_storage_capabilities = one(module.storage_aws_dynamodb[*].capabilities) + + storage_provider_capabilities = local.storage_provider_type == "aws_dynamodb" ? local.dynamodb_storage_capabilities : { + webhook = { + direct = { + environment_variables = tomap({}) + iam_policy_json = null + } + eventbridge = { + webhook = { + environment_variables = tomap({}) + iam_policy_json = null + } + dispatcher = { + environment_variables = tomap({}) + iam_policy_json = null + } + } + } + entries = { + for entry_id in keys(local.effective_config.multi_runner_config) : entry_id => { + scale_up = { + environment_variables = tomap({}) + iam_policy_json = null + } + scale_down = { + environment_variables = tomap({}) + iam_policy_json = null + } + pool = { + environment_variables = tomap({}) + iam_policy_json = null + } + job_retry = { + environment_variables = tomap({}) + iam_policy_json = null + } + runner = { + config_table_name = null + runner_state_table_name = null + scope = null + iam_policy_json = null + } + } + } + } +} diff --git a/modules/multi-runner/variables.experimental.global.tf b/modules/multi-runner/variables.experimental.global.tf index de1def09cf..35c0820593 100644 --- a/modules/multi-runner/variables.experimental.global.tf +++ b/modules/multi-runner/variables.experimental.global.tf @@ -70,3 +70,37 @@ variable "global_config" { }) default = {} } + +variable "global_config_storage_provider" { + description = "Global runner-configuration storage provider selection. Omit the DynamoDB block to retain the existing SSM backend." + type = object({ + aws = optional(object({ + dynamodb = optional(object({ + config = optional(object({ + kms_key_arn = optional(string, null) + point_in_time_recovery_enabled = optional(bool, true) + deletion_protection_enabled = optional(bool, false) + tags = optional(map(string), {}) + }), {}) + runner_state = optional(object({ + kms_key_arn = optional(string, null) + point_in_time_recovery_enabled = optional(bool, false) + deletion_protection_enabled = optional(bool, false) + runner_config_ttl_seconds = optional(number, 86400) + runner_state_ttl_seconds = optional(number, 604800) + tags = optional(map(string), {}) + }), {}) + }), null) + ssm = optional(object({}), null) + }), {}) + }) + default = {} + + validation { + condition = ( + (try(var.global_config_storage_provider.aws.dynamodb, null) != null ? 1 : 0) + + (try(var.global_config_storage_provider.aws.ssm, null) != null ? 1 : 0) + ) <= 1 + error_message = "global_config_storage_provider must select at most one provider: aws.dynamodb or aws.ssm." + } +} diff --git a/modules/multi-runner/webhook.tf b/modules/multi-runner/webhook.tf index f8d16406fe..52ea215f0a 100644 --- a/modules/multi-runner/webhook.tf +++ b/modules/multi-runner/webhook.tf @@ -19,19 +19,24 @@ locals { } } } + + webhook_storage_kms_key_arn = local.storage_provider_type == "aws_ssm" ? local.effective_config.ssm.kms_key_id : null } module "webhook" { source = "../webhook" prefix = var.prefix tags = local.tags - kms_key_arn = local.effective_config.ssm.kms_key_id + kms_key_arn = local.webhook_storage_kms_key_arn eventbridge = { enable = local.effective_config.orchestration_provider.webhook.eventbridge.enabled accept_events = local.effective_config.orchestration_provider.webhook.eventbridge.accept_events } runner_matcher_config = local.runner_matcher_config matcher_config_parameter_store_tier = local.effective_config.orchestration_provider.webhook.matcher_config_parameter_store_tier + storage_provider = merge(local.storage_provider_capabilities.webhook, { + type = local.storage_provider_type + }) ssm_paths = { root = local.ssm_root_path diff --git a/modules/orchestration-providers/webhook/README.md b/modules/orchestration-providers/webhook/README.md index 53d0115ebb..d0af49f902 100644 --- a/modules/orchestration-providers/webhook/README.md +++ b/modules/orchestration-providers/webhook/README.md @@ -47,6 +47,7 @@ The scale-down lifecycle is documented in the [scale-down state diagram](./scale | [runner](#input\_runner) | Common runner registration values consumed by webhook demand controls. Lifecycle, boot timeout, and capacity remain provider-owned under config.runner. | 
object({
    os                   = string
    auto_update_disabled = bool
    labels               = list(string)
    group_name           = string
    name_prefix          = string
  })
| n/a | yes | | [runner\_provider](#input\_runner\_provider) | Selected compute-provider capabilities consumed by webhook scale-up, scale-down, and pool controls. | 
object({
    type = string
    scale_up = object({
      environment_variables      = map(string)
      iam_policy_json            = string
      additional_iam_policy_json = optional(string, null)
      managed_policy = optional(object({
        arn = string
      }), null)
    })
    scale_down = object({
      environment_variables = map(string)
      iam_policy_json       = string
    })
    pool = object({
      environment_variables  = map(string)
      iam_policy_json        = string
      managed_policy_enabled = bool
      managed_policy_arn     = optional(string, null)
    })
  })
| n/a | yes | | [ssm](#input\_ssm) | Resolved Parameter Store paths, optional decrypt key, and runtime parameter tags. | 
object({
    token_path           = string
    token_path_arn       = string
    config_path          = string
    config_path_arn      = string
    kms_key_id           = optional(string, null)
    parameter_store_tags = string
  })
| n/a | yes | +| [storage\_provider](#input\_storage\_provider) | Opaque storage-provider environment and IAM capabilities for webhook control-plane functions. | 
object({
    type = string
    scale_up = object({
      environment_variables = map(string)
      iam_policy_json       = optional(string, null)
    })
    scale_down = object({
      environment_variables = map(string)
      iam_policy_json       = optional(string, null)
    })
    pool = object({
      environment_variables = map(string)
      iam_policy_json       = optional(string, null)
    })
    job_retry = object({
      environment_variables = map(string)
      iam_policy_json       = optional(string, null)
    })
  })
| 
{
  "job_retry": {
    "environment_variables": {},
    "iam_policy_json": null
  },
  "pool": {
    "environment_variables": {},
    "iam_policy_json": null
  },
  "scale_down": {
    "environment_variables": {},
    "iam_policy_json": null
  },
  "scale_up": {
    "environment_variables": {},
    "iam_policy_json": null
  },
  "type": "aws_ssm"
}
| no | | [tags](#input\_tags) | Base tags available to webhook-provider resources. Component-specific tags override this map within their documented scopes. | `map(string)` | `{}` | no | ## Outputs diff --git a/modules/orchestration-providers/webhook/job-retry.tf b/modules/orchestration-providers/webhook/job-retry.tf index 651e12c9ea..fe05707ed1 100644 --- a/modules/orchestration-providers/webhook/job-retry.tf +++ b/modules/orchestration-providers/webhook/job-retry.tf @@ -45,4 +45,9 @@ module "job_retry" { event_source_mapping = local.job_retry_queue_tags } } + + storage_provider = merge( + { type = local.resolved_config.storage_provider.type }, + local.resolved_config.storage_provider.job_retry, + ) } diff --git a/modules/orchestration-providers/webhook/job-retry/README.md b/modules/orchestration-providers/webhook/job-retry/README.md index 9c6e4e0f52..a0dd72e90a 100644 --- a/modules/orchestration-providers/webhook/job-retry/README.md +++ b/modules/orchestration-providers/webhook/job-retry/README.md @@ -53,6 +53,7 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [config](#input\_config) | Provider-neutral job-retry configuration assembled by runner-config.

- `prefix`: Prefix used to name job-retry resources.
- `aws_partition`: AWS partition used to construct the Lambda VPC managed-policy ARN.
- `lambda.artifact.zip`: Resolved local control-plane archive.
- `lambda.artifact.s3.bucket`: Optional S3 bucket containing the Lambda archive.
- `lambda.artifact.s3.key`: Object key of the Lambda archive.
- `lambda.artifact.s3.object_version`: Optional object version of the Lambda archive.
- `lambda.runtime`: Runtime used by the job-retry Lambda.
- `lambda.architecture`: Instruction-set architecture used by the job-retry Lambda.
- `lambda.memory_size`: Memory allocated to the job-retry Lambda.
- `lambda.timeout`: Lambda timeout and retry-queue visibility timeout in seconds.
- `lambda.reserved_concurrent_executions`: Reserved concurrency for the Lambda. Use `-1` for unreserved concurrency.
- `lambda.environment_variables`: Additional Lambda environment variables. Required job-retry variables override matching keys.
- `lambda.vpc.subnet_ids`: Subnets used for Lambda VPC configuration.
- `lambda.vpc.security_group_ids`: Security groups used for Lambda VPC configuration.
- `lambda.role.path`: IAM path used for the job-retry Lambda role.
- `lambda.role.permissions_boundary`: Optional permissions boundary for the Lambda role.
- `lambda.role.principals`: Extra principals allowed to assume the Lambda role, for example during local testing.
- `runner.name_prefix`: Prefix used to identify runners belonging to this runner configuration.
- `github.organization_runners`: Enables organization runners.
- `github.enterprise_server.url`: Optional GitHub Enterprise Server URL.
- `github.enterprise_server.ssl_verify`: Enables TLS certificate verification for GitHub Enterprise Server requests.
- `github.user_agent`: Optional User-Agent sent to GitHub.
- `github.app_parameters.key_base64`: Ordered Parameter Store references for GitHub App private keys.
- `github.app_parameters.id`: Ordered Parameter Store references for GitHub App IDs.
- `github.app_parameters.installation_id`: Ordered optional Parameter Store references for GitHub App installation IDs.
- `queue.build`: URL and ARN of the build queue to which retry messages are published.
- `queue.kms_key_id`: Optional KMS key ARN used to encrypt the build queue. This is distinct from the Parameter Store key.
- `queue.event_source_mapping.batch_size`: Maximum records delivered per job-retry invocation.
- `queue.event_source_mapping.maximum_batching_window_in_seconds`: Maximum event batching window.
- `queue.encryption`: Server-side encryption configuration for the retry queue.
- `ssm.kms_key_id`: Optional KMS key ARN used by the job-retry IAM policy. Its value may be unknown until apply.
- `observability.logs`: Logging level, retention, encryption, and log-class configuration.
- `observability.tracing`: Lambda X-Ray and tracing-helper configuration.
- `observability.metrics`: Metrics enablement, namespace, and job-retry metric configuration.
- `tags.resources`: Tags for the job-retry Lambda role and component resources.
- `tags.lambda`: Tags for the job-retry Lambda function.
- `tags.log_group`: Tags for the job-retry log group.
- `tags.queue`: Tags for the retry queue.
- `tags.event_source_mapping`: Tags for the retry-queue event-source mapping. | 
object({
    prefix        = string
    aws_partition = string
    lambda = object({
      artifact = object({
        zip = string
        s3 = object({
          bucket         = optional(string, null)
          key            = optional(string, null)
          object_version = optional(string, null)
        })
      })
      runtime                        = string
      architecture                   = string
      memory_size                    = number
      timeout                        = number
      reserved_concurrent_executions = number
      environment_variables          = map(string)
      vpc = object({
        subnet_ids         = list(string)
        security_group_ids = list(string)
      })
      role = object({
        path                 = string
        permissions_boundary = optional(string, null)
        principals = list(object({
          type        = string
          identifiers = list(string)
        }))
      })
    })
    runner = object({
      name_prefix = string
    })
    github = object({
      organization_runners = bool
      enterprise_server = object({
        url        = optional(string, null)
        ssl_verify = optional(bool, true)
      })
      user_agent = optional(string, null)
      app_parameters = object({
        key_base64      = list(map(string))
        id              = list(map(string))
        installation_id = list(object({ name = string, arn = string }))
      })
    })
    queue = object({
      build = object({
        url = string
        arn = string
      })
      kms_key_id = optional(string, null)
      event_source_mapping = object({
        batch_size                         = number
        maximum_batching_window_in_seconds = number
      })
      encryption = object({
        sqs_managed_sse_enabled           = bool
        kms_master_key_id                 = optional(string, null)
        kms_data_key_reuse_period_seconds = optional(number, null)
      })
    })
    ssm = object({
      kms_key_id = optional(string, null)
    })
    observability = object({
      logs = object({
        level             = string
        retention_in_days = number
        kms_key_id        = optional(string, null)
        class             = string
      })
      tracing = object({
        mode                  = optional(string, null)
        capture_http_requests = bool
        capture_error         = bool
      })
      metrics = object({
        enabled   = bool
        namespace = string
        metric = object({
          github_app_rate_limit = object({
            enabled = bool
          })
          job_retry = object({
            enabled = bool
          })
        })
      })
    })
    tags = object({
      resources            = map(string)
      lambda               = map(string)
      log_group            = map(string)
      queue                = map(string)
      event_source_mapping = map(string)
    })
  })
| n/a | yes | +| [storage\_provider](#input\_storage\_provider) | Opaque runner-configuration storage capability used by the job-retry Lambda. | 
object({
    type                  = string
    environment_variables = map(string)
    iam_policy_json       = optional(string, null)
  })
| 
{
  "environment_variables": {},
  "iam_policy_json": null,
  "type": "aws_ssm"
}
| no | ## Outputs diff --git a/modules/orchestration-providers/webhook/job-retry/iam-policies.tf b/modules/orchestration-providers/webhook/job-retry/iam-policies.tf index 0e79e8a265..68310119fc 100644 --- a/modules/orchestration-providers/webhook/job-retry/iam-policies.tf +++ b/modules/orchestration-providers/webhook/job-retry/iam-policies.tf @@ -52,20 +52,26 @@ data "aws_iam_policy_document" "lambda_xray" { } data "aws_iam_policy_document" "job_retry" { - statement { - sid = "WebhookJobRetryReadGitHubAppParameters" - effect = "Allow" + source_policy_documents = compact([var.storage_provider.iam_policy_json]) - actions = [ - "ssm:GetParameter", - "ssm:GetParameters", - ] + dynamic "statement" { + for_each = var.storage_provider.type == "aws_ssm" ? [true] : [] + + content { + sid = "WebhookJobRetryReadGitHubAppParameters" + effect = "Allow" + + actions = [ + "ssm:GetParameter", + "ssm:GetParameters", + ] - resources = concat( - [for p in var.config.github.app_parameters.id : p.arn], - [for p in var.config.github.app_parameters.key_base64 : p.arn], - [for p in var.config.github.app_parameters.installation_id : p.arn if p != null], - ) + resources = concat( + [for p in var.config.github.app_parameters.id : p.arn], + [for p in var.config.github.app_parameters.key_base64 : p.arn], + [for p in var.config.github.app_parameters.installation_id : p.arn if p != null], + ) + } } statement { @@ -94,7 +100,7 @@ data "aws_iam_policy_document" "job_retry" { } dynamic "statement" { - for_each = var.config.ssm.kms_key_id == null ? [] : [var.config.ssm.kms_key_id] + for_each = var.storage_provider.type == "aws_ssm" && var.config.ssm.kms_key_id != null ? [var.config.ssm.kms_key_id] : [] iterator = kms_key content { diff --git a/modules/orchestration-providers/webhook/job-retry/job-retry.tf b/modules/orchestration-providers/webhook/job-retry/job-retry.tf index a536cfe196..c33a5aa08c 100644 --- a/modules/orchestration-providers/webhook/job-retry/job-retry.tf +++ b/modules/orchestration-providers/webhook/job-retry/job-retry.tf @@ -19,23 +19,28 @@ locals { } job_retry_environment_variables = { - ENABLE_ORGANIZATION_RUNNERS = var.config.github.organization_runners - ENABLE_METRIC_JOB_RETRY = var.config.observability.metrics.enabled && var.config.observability.metrics.metric.job_retry.enabled - ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = var.config.observability.metrics.enabled && var.config.observability.metrics.metric.github_app_rate_limit.enabled - GHES_URL = var.config.github.enterprise_server.url - NODE_TLS_REJECT_UNAUTHORIZED = var.config.github.enterprise_server.url != null && !var.config.github.enterprise_server.ssl_verify ? 0 : 1 - USER_AGENT = var.config.github.user_agent - JOB_QUEUE_SCALE_UP_URL = var.config.queue.build.url + ENABLE_ORGANIZATION_RUNNERS = var.config.github.organization_runners + ENABLE_METRIC_JOB_RETRY = var.config.observability.metrics.enabled && var.config.observability.metrics.metric.job_retry.enabled + ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = var.config.observability.metrics.enabled && var.config.observability.metrics.metric.github_app_rate_limit.enabled + GHES_URL = var.config.github.enterprise_server.url + NODE_TLS_REJECT_UNAUTHORIZED = var.config.github.enterprise_server.url != null && !var.config.github.enterprise_server.ssl_verify ? 0 : 1 + USER_AGENT = var.config.github.user_agent + JOB_QUEUE_SCALE_UP_URL = var.config.queue.build.url + RUNNER_NAME_PREFIX = var.config.runner.name_prefix + } + + ssm_environment_variables = { PARAMETER_GITHUB_APP_ID_NAME = join(":", [for p in var.config.github.app_parameters.id : p.name]) PARAMETER_GITHUB_APP_KEY_BASE64_NAME = join(":", [for p in var.config.github.app_parameters.key_base64 : p.name]) PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME = join(":", [for p in var.config.github.app_parameters.installation_id : p != null ? p.name : ""]) - RUNNER_NAME_PREFIX = var.config.runner.name_prefix } environment_variables = merge( local.lambda_environment_variables, var.config.lambda.environment_variables, local.job_retry_environment_variables, + var.storage_provider.type == "aws_ssm" ? local.ssm_environment_variables : {}, + var.storage_provider.environment_variables, ) } diff --git a/modules/orchestration-providers/webhook/job-retry/variables.tf b/modules/orchestration-providers/webhook/job-retry/variables.tf index e8235265f8..e7ebc76e37 100644 --- a/modules/orchestration-providers/webhook/job-retry/variables.tf +++ b/modules/orchestration-providers/webhook/job-retry/variables.tf @@ -145,3 +145,17 @@ variable "config" { nullable = false } + +variable "storage_provider" { + description = "Opaque runner-configuration storage capability used by the job-retry Lambda." + type = object({ + type = string + environment_variables = map(string) + iam_policy_json = optional(string, null) + }) + default = { + type = "aws_ssm" + environment_variables = {} + iam_policy_json = null + } +} diff --git a/modules/orchestration-providers/webhook/main.tf b/modules/orchestration-providers/webhook/main.tf index d9b1722d98..8dd566ec18 100644 --- a/modules/orchestration-providers/webhook/main.tf +++ b/modules/orchestration-providers/webhook/main.tf @@ -30,12 +30,13 @@ locals { queue = merge(var.config.queue, { event_source_mapping = var.config.lambda.scale.up.event_source_mapping }) - scale_up = var.config.lambda.scale.up - scale_down = var.config.lambda.scale.down - pool = var.config.lambda.pool - job_retry = var.config.job_retry - ssm = var.ssm - observability = var.observability + scale_up = var.config.lambda.scale.up + scale_down = var.config.lambda.scale.down + pool = var.config.lambda.pool + job_retry = var.config.job_retry + ssm = var.ssm + storage_provider = var.storage_provider + observability = var.observability } common_tags = local.resolved_config.tags diff --git a/modules/orchestration-providers/webhook/pool.tf b/modules/orchestration-providers/webhook/pool.tf index 6fb9ad3d34..4ed40fb919 100644 --- a/modules/orchestration-providers/webhook/pool.tf +++ b/modules/orchestration-providers/webhook/pool.tf @@ -56,6 +56,10 @@ module "pool" { aws_partition = var.aws_partition tracing_config = local.resolved_config.observability.tracing + storage_provider = merge( + { type = local.resolved_config.storage_provider.type }, + local.resolved_config.storage_provider.pool, + ) runner_provider = { type = var.runner_provider.type environment_variables = var.runner_provider.pool.environment_variables diff --git a/modules/orchestration-providers/webhook/pool/README.md b/modules/orchestration-providers/webhook/pool/README.md index 877eec8039..8491aae222 100644 --- a/modules/orchestration-providers/webhook/pool/README.md +++ b/modules/orchestration-providers/webhook/pool/README.md @@ -56,6 +56,7 @@ No modules. | [aws\_partition](#input\_aws\_partition) | (optional) partition for the arn if not 'aws' | `string` | `"aws"` | no | | [config](#input\_config) | Configuration passed from the webhook orchestration provider to the pool Lambda and scheduler.

- `lambda`: Pool Lambda runtime and deployment configuration.
- `lambda.log_level`: Logging level used by the pool Lambda.
- `lambda.logging_retention_in_days`: Number of days to retain events in the pool Lambda log group.
- `lambda.logging_kms_key_id`: KMS key ID used to encrypt the pool Lambda log group.
- `lambda.log_class`: CloudWatch Logs class for the pool Lambda log group.
- `lambda.reserved_concurrent_executions`: Reserved concurrency for the pool Lambda. Use -1 for no reservation.
- `lambda.s3_bucket`: S3 bucket containing the pool Lambda deployment package.
- `lambda.s3_key`: S3 key of the pool Lambda deployment package.
- `lambda.s3_object_version`: S3 object version of the pool Lambda deployment package.
- `lambda.security_group_ids`: Security group IDs associated with the pool Lambda.
- `lambda.runtime`: AWS Lambda runtime used by the pool Lambda.
- `lambda.architecture`: AWS Lambda architecture used by the pool Lambda.
- `lambda.memory_size`: Memory allocated to the pool Lambda in MB.
- `lambda.timeout`: Pool Lambda timeout in seconds.
- `lambda.zip`: Local path to the pool Lambda deployment package when S3 is not used.
- `lambda.subnet_ids`: Subnet IDs in which the pool Lambda runs.
- `lambda.parameter_store_tags`: JSON-encoded tags supplied to the pool Lambda for SSM parameters it creates.
- `lambda.principals`: Additional principals allowed to assume the pool Lambda role.
- `tags`: Common tags added to pool resources.
- `ghes`: GitHub Enterprise Server connection configuration.
- `ghes.url`: GitHub Enterprise Server URL; null when using public GitHub.
- `ghes.ssl_verify`: Whether the pool Lambda verifies the GitHub Enterprise Server TLS certificate.
- `github_app_parameters`: Ordered SSM parameter metadata for GitHub App credentials.
- `github_app_parameters.key_base64`: Ordered Parameter Store references for GitHub App private keys.
- `github_app_parameters.id`: Ordered Parameter Store references for GitHub App IDs.
- `github_app_parameters.installation_id`: Ordered optional Parameter Store references for GitHub App installation IDs.
- `runner`: Runner registration configuration used by the pool Lambda.
- `runner.disable_runner_autoupdate`: Whether GitHub runner automatic updates are disabled.
- `runner.ephemeral`: Whether runners register as ephemeral runners.
- `runner.enable_jit_config`: Whether runners use just-in-time registration configuration.
- `runner.labels`: Labels assigned to runners created by the pool Lambda.
- `runner.group_name`: GitHub runner group assigned to runners created by the pool Lambda.
- `runner.name_prefix`: Prefix used for runner names.
- `runner.pool_owner`: GitHub organization or repository that owns the runner pool.
- `runner.boot_time_in_minutes`: Webhook-provider runner boot timeout used by pool reconciliation.
- `runners_maximum_count`: Webhook-provider runner capacity limit enforced by the pool Lambda.
- `prefix`: Prefix used to name pool resources.
- `pool`: Scheduled pool targets.
- `pool[*].schedule_expression`: EventBridge Scheduler expression for a pool target.
- `pool[*].schedule_expression_timezone`: Time zone used to evaluate the schedule expression.
- `pool[*].size`: Desired runner count for the scheduled pool target.
- `include_busy_runners`: Whether busy runners count toward the desired pool size.
- `role_permissions_boundary`: Permissions boundary applied to IAM roles created for the pool.
- `kms_key_id`: Optional customer-managed KMS key ARN that the pool Lambda may use to decrypt encrypted parameters.
- `role_path`: IAM path applied to roles created for the pool.
- `ssm_token_path`: SSM path under which runner registration tokens are stored.
- `ssm_token_path_arn`: ARN matching the runner registration-token SSM path.
- `ssm_config_path`: SSM path under which runner configuration is stored.
- `arn_ssm_parameters_path_config`: ARN matching the runner configuration SSM path.
- `lambda_tags`: Tags added specifically to the pool Lambda function, overriding common tags with the same key.
- `log_group_tags`: Tags added specifically to the pool Lambda log group, overriding common tags with the same key.
- `user_agent`: User-Agent header used for GitHub API requests. | 
object({
    lambda = object({
      log_level                      = string
      logging_retention_in_days      = number
      logging_kms_key_id             = string
      log_class                      = string
      reserved_concurrent_executions = number
      s3_bucket                      = string
      s3_key                         = string
      s3_object_version              = string
      security_group_ids             = list(string)
      runtime                        = string
      architecture                   = string
      memory_size                    = number
      timeout                        = number
      zip                            = string
      subnet_ids                     = list(string)
      parameter_store_tags           = string
      principals = optional(list(object({
        type        = string
        identifiers = list(string)
      })), [])
    })
    tags = map(string)
    ghes = object({
      url        = string
      ssl_verify = string
    })
    github_app_parameters = object({
      key_base64      = list(map(string))
      id              = list(map(string))
      installation_id = list(object({ name = string, arn = string }))
    })
    runner = object({
      disable_runner_autoupdate = bool
      ephemeral                 = bool
      enable_jit_config         = bool
      labels                    = list(string)
      group_name                = string
      name_prefix               = string
      pool_owner                = string
      boot_time_in_minutes      = number
    })
    runners_maximum_count = number
    prefix                = string
    pool = list(object({
      schedule_expression          = string
      schedule_expression_timezone = string
      size                         = number
    }))
    include_busy_runners           = bool
    role_permissions_boundary      = string
    kms_key_id                     = optional(string, null)
    role_path                      = string
    ssm_token_path                 = string
    ssm_token_path_arn             = string
    ssm_config_path                = string
    arn_ssm_parameters_path_config = string
    lambda_tags                    = map(string)
    log_group_tags                 = optional(map(string), {})
    user_agent                     = string
  })
| n/a | yes | | [runner\_provider](#input\_runner\_provider) | Compute provider integration used by the pool Lambda.

- `type`: Compute provider type passed to scheduled pool invocations.
- `environment_variables`: Provider-specific environment variables added to the pool Lambda.
- `iam_policy_json`: Provider-specific IAM policy document merged into the pool Lambda policy.
- `managed_policy_enabled`: Whether to attach a provider-specific managed IAM policy to the pool Lambda role.
- `managed_policy_arn`: ARN of the provider-specific managed IAM policy to attach when enabled. | 
object({
    type                   = string
    environment_variables  = map(string)
    iam_policy_json        = string
    managed_policy_enabled = bool
    managed_policy_arn     = optional(string, null)
  })
| n/a | yes | +| [storage\_provider](#input\_storage\_provider) | Opaque runner-configuration storage capability used by the pool Lambda. | 
object({
    type                  = string
    environment_variables = map(string)
    iam_policy_json       = optional(string, null)
  })
| 
{
  "environment_variables": {},
  "iam_policy_json": null,
  "type": "aws_ssm"
}
| no | | [tracing\_config](#input\_tracing\_config) | Tracing configuration for the pool Lambda.

- `mode`: AWS X-Ray tracing mode. A null value disables tracing.
- `capture_http_requests`: Whether Powertools tracing captures outgoing HTTP requests.
- `capture_error`: Whether Powertools tracing captures errors as tracing metadata. | 
object({
    mode                  = optional(string, null)
    capture_http_requests = optional(bool, false)
    capture_error         = optional(bool, false)
  })
| `{}` | no | ## Outputs diff --git a/modules/orchestration-providers/webhook/pool/iam-policies.tf b/modules/orchestration-providers/webhook/pool/iam-policies.tf index f5a9285bce..334aa2845c 100644 --- a/modules/orchestration-providers/webhook/pool/iam-policies.tf +++ b/modules/orchestration-providers/webhook/pool/iam-policies.tf @@ -1,56 +1,68 @@ # IAM policies attached to the pool Lambda role. data "aws_iam_policy_document" "pool_common" { - statement { - sid = "WebhookPoolWriteRuntimeParameters" - effect = "Allow" + dynamic "statement" { + for_each = var.storage_provider.type == "aws_ssm" ? [true] : [] - actions = [ - "ssm:AddTagsToResource", - "ssm:PutParameter", - ] + content { + sid = "WebhookPoolWriteRuntimeParameters" + effect = "Allow" - resources = [ - var.config.ssm_token_path_arn, - "${var.config.ssm_token_path_arn}/*", - var.config.arn_ssm_parameters_path_config, - "${var.config.arn_ssm_parameters_path_config}/*", - ] + actions = [ + "ssm:AddTagsToResource", + "ssm:PutParameter", + ] + + resources = [ + var.config.ssm_token_path_arn, + "${var.config.ssm_token_path_arn}/*", + var.config.arn_ssm_parameters_path_config, + "${var.config.arn_ssm_parameters_path_config}/*", + ] + } } - statement { - sid = "WebhookPoolReadRunnerConfigParameters" - effect = "Allow" + dynamic "statement" { + for_each = var.storage_provider.type == "aws_ssm" ? [true] : [] - actions = [ - "ssm:GetParameter", - "ssm:GetParameters", - "ssm:GetParametersByPath", - ] + content { + sid = "WebhookPoolReadRunnerConfigParameters" + effect = "Allow" - resources = [ - var.config.arn_ssm_parameters_path_config, - "${var.config.arn_ssm_parameters_path_config}/*", - ] + actions = [ + "ssm:GetParameter", + "ssm:GetParameters", + "ssm:GetParametersByPath", + ] + + resources = [ + var.config.arn_ssm_parameters_path_config, + "${var.config.arn_ssm_parameters_path_config}/*", + ] + } } - statement { - sid = "WebhookPoolReadGitHubAppParameters" - effect = "Allow" + dynamic "statement" { + for_each = var.storage_provider.type == "aws_ssm" ? [true] : [] - actions = [ - "ssm:GetParameter", - "ssm:GetParameters", - ] + content { + sid = "WebhookPoolReadGitHubAppParameters" + effect = "Allow" + + actions = [ + "ssm:GetParameter", + "ssm:GetParameters", + ] - resources = concat( - [for p in var.config.github_app_parameters.id : p.arn], - [for p in var.config.github_app_parameters.key_base64 : p.arn], - [for p in var.config.github_app_parameters.installation_id : p.arn if p != null], - ) + resources = concat( + [for p in var.config.github_app_parameters.id : p.arn], + [for p in var.config.github_app_parameters.key_base64 : p.arn], + [for p in var.config.github_app_parameters.installation_id : p.arn if p != null], + ) + } } dynamic "statement" { - for_each = var.config.kms_key_id == null ? [] : [var.config.kms_key_id] + for_each = var.storage_provider.type == "aws_ssm" && var.config.kms_key_id != null ? [var.config.kms_key_id] : [] iterator = kms_key content { diff --git a/modules/orchestration-providers/webhook/pool/pool.tf b/modules/orchestration-providers/webhook/pool/pool.tf index cff2776e90..ce3319c8c8 100644 --- a/modules/orchestration-providers/webhook/pool/pool.tf +++ b/modules/orchestration-providers/webhook/pool/pool.tf @@ -7,32 +7,35 @@ locals { ) common_environment_variables = { - DISABLE_RUNNER_AUTOUPDATE = var.config.runner.disable_runner_autoupdate - ENABLE_EPHEMERAL_RUNNERS = var.config.runner.ephemeral - ENABLE_JIT_CONFIG = var.config.runner.enable_jit_config - ENVIRONMENT = var.config.prefix - GHES_URL = var.config.ghes.url - USER_AGENT = var.config.user_agent - LOG_LEVEL = upper(var.config.lambda.log_level) - NODE_TLS_REJECT_UNAUTHORIZED = var.config.ghes.url != null && !var.config.ghes.ssl_verify ? 0 : 1 + DISABLE_RUNNER_AUTOUPDATE = var.config.runner.disable_runner_autoupdate + ENABLE_EPHEMERAL_RUNNERS = var.config.runner.ephemeral + ENABLE_JIT_CONFIG = var.config.runner.enable_jit_config + ENVIRONMENT = var.config.prefix + GHES_URL = var.config.ghes.url + USER_AGENT = var.config.user_agent + LOG_LEVEL = upper(var.config.lambda.log_level) + NODE_TLS_REJECT_UNAUTHORIZED = var.config.ghes.url != null && !var.config.ghes.ssl_verify ? 0 : 1 + POWERTOOLS_LOGGER_LOG_EVENT = var.config.lambda.log_level == "debug" ? "true" : "false" + RUNNER_LABELS = lower(join(",", var.config.runner.labels)) + RUNNER_GROUP_NAME = var.config.runner.group_name + RUNNER_NAME_PREFIX = var.config.runner.name_prefix + RUNNER_OWNER = var.config.runner.pool_owner + RUNNER_BOOT_TIME_IN_MINUTES = var.config.runner.boot_time_in_minutes + RUNNERS_MAXIMUM_COUNT = var.config.runners_maximum_count + POWERTOOLS_SERVICE_NAME = "${var.config.prefix}-pool" + POWERTOOLS_TRACE_ENABLED = var.tracing_config.mode != null ? true : false + POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.tracing_config.capture_http_requests + POWERTOOLS_TRACER_CAPTURE_ERROR = var.tracing_config.capture_error + INCLUDE_BUSY_RUNNERS = var.config.include_busy_runners + } + + ssm_environment_variables = { PARAMETER_GITHUB_APP_ID_NAME = join(":", [for p in var.config.github_app_parameters.id : p.name]) PARAMETER_GITHUB_APP_KEY_BASE64_NAME = join(":", [for p in var.config.github_app_parameters.key_base64 : p.name]) PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME = join(":", [for p in var.config.github_app_parameters.installation_id : p != null ? p.name : ""]) - POWERTOOLS_LOGGER_LOG_EVENT = var.config.lambda.log_level == "debug" ? "true" : "false" - RUNNER_LABELS = lower(join(",", var.config.runner.labels)) - RUNNER_GROUP_NAME = var.config.runner.group_name - RUNNER_NAME_PREFIX = var.config.runner.name_prefix - RUNNER_OWNER = var.config.runner.pool_owner - RUNNER_BOOT_TIME_IN_MINUTES = var.config.runner.boot_time_in_minutes - RUNNERS_MAXIMUM_COUNT = var.config.runners_maximum_count SSM_TOKEN_PATH = var.config.ssm_token_path SSM_CONFIG_PATH = var.config.ssm_config_path - POWERTOOLS_SERVICE_NAME = "${var.config.prefix}-pool" - POWERTOOLS_TRACE_ENABLED = var.tracing_config.mode != null ? true : false - POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.tracing_config.capture_http_requests - POWERTOOLS_TRACER_CAPTURE_ERROR = var.tracing_config.capture_error SSM_PARAMETER_STORE_TAGS = var.config.lambda.parameter_store_tags - INCLUDE_BUSY_RUNNERS = var.config.include_busy_runners } } @@ -54,7 +57,12 @@ resource "aws_lambda_function" "pool" { tags = merge(var.config.tags, var.config.lambda_tags) environment { - variables = merge(var.runner_provider.environment_variables, local.common_environment_variables) + variables = merge( + var.runner_provider.environment_variables, + local.common_environment_variables, + var.storage_provider.type == "aws_ssm" ? local.ssm_environment_variables : {}, + var.storage_provider.environment_variables, + ) } dynamic "vpc_config" { @@ -96,10 +104,11 @@ resource "aws_iam_role_policy" "pool" { } data "aws_iam_policy_document" "pool" { - source_policy_documents = [ + source_policy_documents = compact([ data.aws_iam_policy_document.pool_common.json, var.runner_provider.iam_policy_json, - ] + var.storage_provider.iam_policy_json, + ]) } resource "aws_iam_role_policy" "pool_logging" { diff --git a/modules/orchestration-providers/webhook/pool/variables.tf b/modules/orchestration-providers/webhook/pool/variables.tf index e1f516c8ad..2b3b9cfb0b 100644 --- a/modules/orchestration-providers/webhook/pool/variables.tf +++ b/modules/orchestration-providers/webhook/pool/variables.tf @@ -138,6 +138,20 @@ variable "runner_provider" { }) } +variable "storage_provider" { + description = "Opaque runner-configuration storage capability used by the pool Lambda." + type = object({ + type = string + environment_variables = map(string) + iam_policy_json = optional(string, null) + }) + default = { + type = "aws_ssm" + environment_variables = {} + iam_policy_json = null + } +} + variable "aws_partition" { description = "(optional) partition for the arn if not 'aws'" type = string diff --git a/modules/orchestration-providers/webhook/scale-runners.tf b/modules/orchestration-providers/webhook/scale-runners.tf index caf79eefb5..cdcbc9350a 100644 --- a/modules/orchestration-providers/webhook/scale-runners.tf +++ b/modules/orchestration-providers/webhook/scale-runners.tf @@ -53,6 +53,12 @@ module "scale_runners" { } } + storage_provider = { + type = local.resolved_config.storage_provider.type + scale_up = local.resolved_config.storage_provider.scale_up + scale_down = local.resolved_config.storage_provider.scale_down + } + runner_provider = { type = var.runner_provider.type scale_up = var.runner_provider.scale_up diff --git a/modules/orchestration-providers/webhook/scale-runners/README.md b/modules/orchestration-providers/webhook/scale-runners/README.md index 3b096f9b85..36f92f12dd 100644 --- a/modules/orchestration-providers/webhook/scale-runners/README.md +++ b/modules/orchestration-providers/webhook/scale-runners/README.md @@ -69,6 +69,7 @@ No modules. | [aws\_partition](#input\_aws\_partition) | AWS partition used to construct IAM policy ARNs. | `string` | `"aws"` | no | | [config](#input\_config) | Provider-neutral scale-up and scale-down configuration assembled by runner-config.

- `prefix`: Prefix used to name scaling resources.
- `lambda.artifact.zip`: Resolved local control-plane archive.
- `lambda.artifact.s3.bucket`: Optional S3 bucket containing the Lambda archive.
- `lambda.artifact.s3.key`: Object key of the Lambda archive.
- `lambda.artifact.s3.object_version`: Optional object version of the Lambda archive.
- `lambda.runtime`: Runtime used by both scaling Lambdas.
- `lambda.architecture`: Instruction-set architecture used by both scaling Lambdas.
- `lambda.vpc.subnet_ids`: Subnets used for Lambda VPC configuration.
- `lambda.vpc.security_group_ids`: Security groups used for Lambda VPC configuration.
- `lambda.role.path`: IAM path used for the scaling Lambda roles.
- `lambda.role.permissions_boundary`: Optional permissions boundary for the scaling Lambda roles.
- `lambda.role.principals`: Additional principals allowed to assume the scaling Lambda roles.
- `runner.os`: Runner operating system used for the minimum-runtime default.
- `runner.auto_update_disabled`: Disables the GitHub runner application's built-in updater.
- `runner.ephemeral`: Registers runners in ephemeral mode.
- `runner.jit_config_enabled`: Enables or disables just-in-time runner configuration.
- `runner.labels`: Labels supplied when a runner is registered.
- `runner.group_name`: GitHub runner group used during registration.
- `runner.name_prefix`: Prefix added to registered runner names.
- `runner.boot_time_in_minutes`: Webhook-provider runner boot timeout used by scale-down.
- `runner.maximum_count`: Webhook-provider runner capacity limit for this runner configuration.
- `github.organization_runners`: Registers organization runners when true.
- `github.enterprise_server.url`: Optional GitHub Enterprise Server URL.
- `github.enterprise_server.ssl_verify`: Enables TLS verification for GitHub Enterprise Server.
- `github.user_agent`: Optional User-Agent sent to GitHub.
- `github.app_parameters.key_base64`: Ordered Parameter Store references for GitHub App private keys.
- `github.app_parameters.id`: Ordered Parameter Store references for GitHub App IDs.
- `github.app_parameters.installation_id`: Ordered optional Parameter Store references for GitHub App installation IDs.
- `queue.build.arn`: ARN of the build queue consumed by scale-up.
- `queue.kms_key_id`: Optional KMS key ARN used to encrypt the build queue. This is distinct from the Parameter Store key.
- `queue.event_source_mapping.batch_size`: Maximum records delivered per scale-up invocation.
- `queue.event_source_mapping.maximum_batching_window_in_seconds`: Maximum event batching window.
- `ssm.token_path`: Parameter Store path used for registration tokens.
- `ssm.token_path_arn`: ARN of the Parameter Store path used for registration tokens.
- `ssm.config_path`: Parameter Store path used for persistent runner configuration.
- `ssm.config_path_arn`: ARN of the persistent runner configuration path.
- `ssm.kms_key_id`: Optional KMS key ARN used to decrypt shared parameters. Its value may be unknown until apply.
- `ssm.parameter_store_tags`: JSON-encoded tags applied to parameters created at runtime.
- `observability.logs`: Shared logging level, retention, encryption, and log-class configuration.
- `observability.tracing`: Lambda X-Ray and tracing-helper configuration.
- `observability.metrics`: Metrics enablement, namespace, and GitHub rate-limit metric configuration.
- `scale_up`: Scale-up Lambda sizing, concurrency, queued-job behavior, and resolved resource tag maps.
- `scale_up.tags.resources`: Tags for the scale-up IAM role and other component resources.
- `scale_up.tags.lambda`: Tags for the scale-up Lambda function.
- `scale_up.tags.log_group`: Tags for the scale-up log group.
- `scale_up.tags.event_source_mapping`: Tags for the build-queue event-source mapping.
- `scale_down`: Scale-down Lambda sizing, schedule, idle configuration, minimum runtime, and resolved resource tag maps.
- `scale_down.tags.resources`: Tags for the scale-down IAM role and EventBridge rule.
- `scale_down.tags.lambda`: Tags for the scale-down Lambda function.
- `scale_down.tags.log_group`: Tags for the scale-down log group.
- `job_retry.enabled`: Enables publishing retry checks from scale-up.
- `job_retry.queue`: Retry queue ARN and URL. Required when job retry is enabled.
- `job_retry.max_attempts`: Maximum queued-job retry attempts.
- `job_retry.delay_in_seconds`: Initial delay before checking the queued job.
- `job_retry.delay_backoff`: Multiplier applied to subsequent delays. | 
object({
    prefix = string
    lambda = object({
      artifact = object({
        zip = string
        s3 = object({
          bucket         = optional(string, null)
          key            = optional(string, null)
          object_version = optional(string, null)
        })
      })
      runtime      = string
      architecture = string
      vpc = object({
        subnet_ids         = list(string)
        security_group_ids = list(string)
      })
      role = object({
        path                 = string
        permissions_boundary = optional(string, null)
        principals = optional(list(object({
          type        = string
          identifiers = list(string)
        })), [])
      })
    })
    runner = object({
      os                   = string
      auto_update_disabled = bool
      ephemeral            = bool
      jit_config_enabled   = optional(bool, null)
      labels               = list(string)
      group_name           = string
      name_prefix          = string
      boot_time_in_minutes = number
      maximum_count        = number
    })
    github = object({
      organization_runners = bool
      enterprise_server = object({
        url        = optional(string, null)
        ssl_verify = bool
      })
      user_agent = optional(string, null)
      app_parameters = object({
        key_base64      = list(map(string))
        id              = list(map(string))
        installation_id = list(object({ name = string, arn = string }))
      })
    })
    queue = object({
      build = object({
        arn = string
      })
      kms_key_id = optional(string, null)
      event_source_mapping = object({
        batch_size                         = number
        maximum_batching_window_in_seconds = number
      })
    })
    ssm = object({
      token_path           = string
      token_path_arn       = string
      config_path          = string
      config_path_arn      = string
      parameter_store_tags = string
      kms_key_id           = optional(string, null)
    })
    observability = object({
      logs = object({
        level             = string
        retention_in_days = number
        kms_key_id        = optional(string, null)
        class             = string
      })
      tracing = object({
        mode                  = optional(string, null)
        capture_http_requests = bool
        capture_error         = bool
      })
      metrics = object({
        enabled   = bool
        namespace = string
        metric = object({
          github_app_rate_limit = object({
            enabled = bool
          })
        })
      })
    })
    scale_up = object({
      memory_size                    = number
      timeout                        = number
      reserved_concurrent_executions = number
      job_queued_check_enabled       = bool
      tags = object({
        resources            = map(string)
        lambda               = map(string)
        log_group            = map(string)
        event_source_mapping = map(string)
      })
    })
    scale_down = object({
      memory_size                     = number
      timeout                         = number
      schedule_expression             = string
      minimum_running_time_in_minutes = optional(number, null)
      idle_config = list(object({
        cron             = string
        timeZone         = string
        idleCount        = number
        evictionStrategy = string
      }))
      tags = object({
        resources = map(string)
        lambda    = map(string)
        log_group = map(string)
      })
    })
    job_retry = object({
      enabled          = bool
      max_attempts     = number
      delay_in_seconds = number
      delay_backoff    = number
      queue = optional(object({
        arn = string
        url = string
      }), null)
    })
  })
| n/a | yes | | [runner\_provider](#input\_runner\_provider) | Selected compute-provider integration for the scaling control plane.

- `type`: Compute-provider discriminator supplied to both Lambdas.
- `scale_up.environment_variables`: Provider-specific scale-up environment variables.
- `scale_up.iam_policy_json`: Provider-specific IAM policy merged into the common scale-up policy.
- `scale_up.additional_iam_policy_json`: Optional additional provider policy attached separately to the scale-up role.
- `scale_up.managed_policy`: Optional provider-managed policy attachment. Object presence controls attachment creation.
- `scale_up.managed_policy.arn`: ARN of the provider-managed policy. The ARN may remain unknown until apply.
- `scale_down.environment_variables`: Provider-specific scale-down environment variables.
- `scale_down.iam_policy_json`: Provider-specific IAM policy merged into the common scale-down policy. | 
object({
    type = string
    scale_up = object({
      environment_variables      = map(string)
      iam_policy_json            = string
      additional_iam_policy_json = optional(string, null)
      managed_policy = optional(object({
        arn = string
      }), null)
    })
    scale_down = object({
      environment_variables = map(string)
      iam_policy_json       = string
    })
  })
| n/a | yes | +| [storage\_provider](#input\_storage\_provider) | Opaque storage-provider capabilities for scale-up and scale-down. | 
object({
    type = string
    scale_up = object({
      environment_variables = map(string)
      iam_policy_json       = optional(string, null)
    })
    scale_down = object({
      environment_variables = map(string)
      iam_policy_json       = optional(string, null)
    })
  })
| 
{
  "scale_down": {
    "environment_variables": {},
    "iam_policy_json": null
  },
  "scale_up": {
    "environment_variables": {},
    "iam_policy_json": null
  },
  "type": "aws_ssm"
}
| no | ## Outputs diff --git a/modules/orchestration-providers/webhook/scale-runners/scale-down-iam-policies.tf b/modules/orchestration-providers/webhook/scale-runners/scale-down-iam-policies.tf index b95cb9e686..79571c2747 100644 --- a/modules/orchestration-providers/webhook/scale-runners/scale-down-iam-policies.tf +++ b/modules/orchestration-providers/webhook/scale-runners/scale-down-iam-policies.tf @@ -1,20 +1,24 @@ data "aws_iam_policy_document" "scale_down_common" { - statement { - sid = "WebhookScaleDownReadGitHubAppParameters" - effect = "Allow" - actions = [ - "ssm:GetParameter", - "ssm:GetParameters", - ] - resources = concat( - [for p in var.config.github.app_parameters.id : p.arn], - [for p in var.config.github.app_parameters.key_base64 : p.arn], - [for p in var.config.github.app_parameters.installation_id : p.arn if p != null], - ) + dynamic "statement" { + for_each = var.storage_provider.type == "aws_ssm" ? [true] : [] + + content { + sid = "WebhookScaleDownReadGitHubAppParameters" + effect = "Allow" + actions = [ + "ssm:GetParameter", + "ssm:GetParameters", + ] + resources = concat( + [for p in var.config.github.app_parameters.id : p.arn], + [for p in var.config.github.app_parameters.key_base64 : p.arn], + [for p in var.config.github.app_parameters.installation_id : p.arn if p != null], + ) + } } dynamic "statement" { - for_each = var.config.ssm.kms_key_id == null ? [] : [var.config.ssm.kms_key_id] + for_each = var.storage_provider.type == "aws_ssm" && var.config.ssm.kms_key_id != null ? [var.config.ssm.kms_key_id] : [] iterator = kms_key content { @@ -27,10 +31,11 @@ data "aws_iam_policy_document" "scale_down_common" { } data "aws_iam_policy_document" "scale_down" { - source_policy_documents = [ + source_policy_documents = compact([ data.aws_iam_policy_document.scale_down_common.json, var.runner_provider.scale_down.iam_policy_json, - ] + var.storage_provider.scale_down.iam_policy_json, + ]) } data "aws_iam_policy_document" "scale_down_logging" { diff --git a/modules/orchestration-providers/webhook/scale-runners/scale-down.tf b/modules/orchestration-providers/webhook/scale-runners/scale-down.tf index 44e651d78c..7c5c19f4fe 100644 --- a/modules/orchestration-providers/webhook/scale-runners/scale-down.tf +++ b/modules/orchestration-providers/webhook/scale-runners/scale-down.tf @@ -15,26 +15,27 @@ resource "aws_lambda_function" "scale_down" { environment { variables = merge(var.runner_provider.scale_down.environment_variables, { - ENVIRONMENT = var.config.prefix - ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = var.config.observability.metrics.enabled && var.config.observability.metrics.metric.github_app_rate_limit.enabled - GHES_URL = var.config.github.enterprise_server.url - USER_AGENT = var.config.github.user_agent - LOG_LEVEL = upper(var.config.observability.logs.level) - MINIMUM_RUNNING_TIME_IN_MINUTES = coalesce(var.config.scale_down.minimum_running_time_in_minutes, local.min_runtime_defaults[var.config.runner.os]) - NODE_TLS_REJECT_UNAUTHORIZED = var.config.github.enterprise_server.url != null && !var.config.github.enterprise_server.ssl_verify ? 0 : 1 + ENVIRONMENT = var.config.prefix + ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = var.config.observability.metrics.enabled && var.config.observability.metrics.metric.github_app_rate_limit.enabled + GHES_URL = var.config.github.enterprise_server.url + USER_AGENT = var.config.github.user_agent + LOG_LEVEL = upper(var.config.observability.logs.level) + MINIMUM_RUNNING_TIME_IN_MINUTES = coalesce(var.config.scale_down.minimum_running_time_in_minutes, local.min_runtime_defaults[var.config.runner.os]) + NODE_TLS_REJECT_UNAUTHORIZED = var.config.github.enterprise_server.url != null && !var.config.github.enterprise_server.ssl_verify ? 0 : 1 + POWERTOOLS_LOGGER_LOG_EVENT = var.config.observability.logs.level == "debug" ? "true" : "false" + SCALE_DOWN_CONFIG = jsonencode(var.config.scale_down.idle_config) + POWERTOOLS_SERVICE_NAME = "${var.config.prefix}-scale-down" + POWERTOOLS_METRICS_NAMESPACE = var.config.observability.metrics.namespace + POWERTOOLS_TRACE_ENABLED = var.config.observability.tracing.mode != null + POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.config.observability.tracing.capture_http_requests + POWERTOOLS_TRACER_CAPTURE_ERROR = var.config.observability.tracing.capture_error + COMPUTE_PROVIDER_TYPE = var.runner_provider.type + RUNNER_BOOT_TIME_IN_MINUTES = var.config.runner.boot_time_in_minutes + }, var.storage_provider.type == "aws_ssm" ? { PARAMETER_GITHUB_APP_ID_NAME = join(":", [for p in var.config.github.app_parameters.id : p.name]) PARAMETER_GITHUB_APP_KEY_BASE64_NAME = join(":", [for p in var.config.github.app_parameters.key_base64 : p.name]) PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME = join(":", [for p in var.config.github.app_parameters.installation_id : p != null ? p.name : ""]) - POWERTOOLS_LOGGER_LOG_EVENT = var.config.observability.logs.level == "debug" ? "true" : "false" - SCALE_DOWN_CONFIG = jsonencode(var.config.scale_down.idle_config) - POWERTOOLS_SERVICE_NAME = "${var.config.prefix}-scale-down" - POWERTOOLS_METRICS_NAMESPACE = var.config.observability.metrics.namespace - POWERTOOLS_TRACE_ENABLED = var.config.observability.tracing.mode != null - POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.config.observability.tracing.capture_http_requests - POWERTOOLS_TRACER_CAPTURE_ERROR = var.config.observability.tracing.capture_error - COMPUTE_PROVIDER_TYPE = var.runner_provider.type - RUNNER_BOOT_TIME_IN_MINUTES = var.config.runner.boot_time_in_minutes - }) + } : {}, var.storage_provider.scale_down.environment_variables) } dynamic "vpc_config" { diff --git a/modules/orchestration-providers/webhook/scale-runners/scale-up-iam-policies.tf b/modules/orchestration-providers/webhook/scale-runners/scale-up-iam-policies.tf index b3c87b8ad7..3fddefbe99 100644 --- a/modules/orchestration-providers/webhook/scale-runners/scale-up-iam-policies.tf +++ b/modules/orchestration-providers/webhook/scale-runners/scale-up-iam-policies.tf @@ -1,35 +1,43 @@ data "aws_iam_policy_document" "scale_up_common" { - statement { - sid = "WebhookScaleUpWriteRuntimeParameters" - effect = "Allow" - actions = [ - "ssm:PutParameter", - "ssm:AddTagsToResource", - ] - resources = [ - var.config.ssm.token_path_arn, - "${var.config.ssm.token_path_arn}/*", - var.config.ssm.config_path_arn, - "${var.config.ssm.config_path_arn}/*", - ] - } + dynamic "statement" { + for_each = var.storage_provider.type == "aws_ssm" ? [true] : [] - statement { - sid = "WebhookScaleUpReadGitHubAppAndRunnerConfigParameters" - effect = "Allow" - actions = [ - "ssm:GetParameter", - "ssm:GetParameters", - ] - resources = concat( - [for p in var.config.github.app_parameters.id : p.arn], - [for p in var.config.github.app_parameters.key_base64 : p.arn], - [for p in var.config.github.app_parameters.installation_id : p.arn if p != null], - [ + content { + sid = "WebhookScaleUpWriteRuntimeParameters" + effect = "Allow" + actions = [ + "ssm:PutParameter", + "ssm:AddTagsToResource", + ] + resources = [ + var.config.ssm.token_path_arn, + "${var.config.ssm.token_path_arn}/*", var.config.ssm.config_path_arn, "${var.config.ssm.config_path_arn}/*", - ], - ) + ] + } + } + + dynamic "statement" { + for_each = var.storage_provider.type == "aws_ssm" ? [true] : [] + + content { + sid = "WebhookScaleUpReadGitHubAppAndRunnerConfigParameters" + effect = "Allow" + actions = [ + "ssm:GetParameter", + "ssm:GetParameters", + ] + resources = concat( + [for p in var.config.github.app_parameters.id : p.arn], + [for p in var.config.github.app_parameters.key_base64 : p.arn], + [for p in var.config.github.app_parameters.installation_id : p.arn if p != null], + [ + var.config.ssm.config_path_arn, + "${var.config.ssm.config_path_arn}/*", + ], + ) + } } statement { @@ -44,7 +52,7 @@ data "aws_iam_policy_document" "scale_up_common" { } dynamic "statement" { - for_each = var.config.ssm.kms_key_id == null ? [] : [var.config.ssm.kms_key_id] + for_each = var.storage_provider.type == "aws_ssm" && var.config.ssm.kms_key_id != null ? [var.config.ssm.kms_key_id] : [] iterator = kms_key content { @@ -69,10 +77,11 @@ data "aws_iam_policy_document" "scale_up_common" { } data "aws_iam_policy_document" "scale_up" { - source_policy_documents = [ + source_policy_documents = compact([ data.aws_iam_policy_document.scale_up_common.json, var.runner_provider.scale_up.iam_policy_json, - ] + var.storage_provider.scale_up.iam_policy_json, + ]) } data "aws_iam_policy_document" "scale_up_logging" { diff --git a/modules/orchestration-providers/webhook/scale-runners/scale-up.tf b/modules/orchestration-providers/webhook/scale-runners/scale-up.tf index 2997aeac21..43ac0aff76 100644 --- a/modules/orchestration-providers/webhook/scale-runners/scale-up.tf +++ b/modules/orchestration-providers/webhook/scale-runners/scale-up.tf @@ -16,37 +16,38 @@ resource "aws_lambda_function" "scale_up" { environment { variables = merge(var.runner_provider.scale_up.environment_variables, { - DISABLE_RUNNER_AUTOUPDATE = var.config.runner.auto_update_disabled - ENABLE_EPHEMERAL_RUNNERS = var.config.runner.ephemeral - ENABLE_JIT_CONFIG = var.config.runner.jit_config_enabled - ENABLE_JOB_QUEUED_CHECK = var.config.scale_up.job_queued_check_enabled - ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = var.config.observability.metrics.enabled && var.config.observability.metrics.metric.github_app_rate_limit.enabled - ENABLE_ORGANIZATION_RUNNERS = var.config.github.organization_runners - ENVIRONMENT = var.config.prefix - GHES_URL = var.config.github.enterprise_server.url - USER_AGENT = var.config.github.user_agent - LOG_LEVEL = upper(var.config.observability.logs.level) - MINIMUM_RUNNING_TIME_IN_MINUTES = coalesce(var.config.scale_down.minimum_running_time_in_minutes, local.min_runtime_defaults[var.config.runner.os]) - NODE_TLS_REJECT_UNAUTHORIZED = var.config.github.enterprise_server.url != null && !var.config.github.enterprise_server.ssl_verify ? 0 : 1 + DISABLE_RUNNER_AUTOUPDATE = var.config.runner.auto_update_disabled + ENABLE_EPHEMERAL_RUNNERS = var.config.runner.ephemeral + ENABLE_JIT_CONFIG = var.config.runner.jit_config_enabled + ENABLE_JOB_QUEUED_CHECK = var.config.scale_up.job_queued_check_enabled + ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = var.config.observability.metrics.enabled && var.config.observability.metrics.metric.github_app_rate_limit.enabled + ENABLE_ORGANIZATION_RUNNERS = var.config.github.organization_runners + ENVIRONMENT = var.config.prefix + GHES_URL = var.config.github.enterprise_server.url + USER_AGENT = var.config.github.user_agent + LOG_LEVEL = upper(var.config.observability.logs.level) + MINIMUM_RUNNING_TIME_IN_MINUTES = coalesce(var.config.scale_down.minimum_running_time_in_minutes, local.min_runtime_defaults[var.config.runner.os]) + NODE_TLS_REJECT_UNAUTHORIZED = var.config.github.enterprise_server.url != null && !var.config.github.enterprise_server.ssl_verify ? 0 : 1 + POWERTOOLS_LOGGER_LOG_EVENT = var.config.observability.logs.level == "debug" ? "true" : "false" + POWERTOOLS_METRICS_NAMESPACE = var.config.observability.metrics.namespace + POWERTOOLS_TRACE_ENABLED = var.config.observability.tracing.mode != null + POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.config.observability.tracing.capture_http_requests + POWERTOOLS_TRACER_CAPTURE_ERROR = var.config.observability.tracing.capture_error + RUNNER_LABELS = lower(join(",", var.config.runner.labels)) + RUNNER_GROUP_NAME = var.config.runner.group_name + RUNNER_NAME_PREFIX = var.config.runner.name_prefix + COMPUTE_PROVIDER_TYPE = var.runner_provider.type + RUNNERS_MAXIMUM_COUNT = var.config.runner.maximum_count + POWERTOOLS_SERVICE_NAME = "${var.config.prefix}-scale-up" + JOB_RETRY_CONFIG = jsonencode(local.job_retry_config) + }, var.storage_provider.type == "aws_ssm" ? { PARAMETER_GITHUB_APP_ID_NAME = join(":", [for p in var.config.github.app_parameters.id : p.name]) PARAMETER_GITHUB_APP_KEY_BASE64_NAME = join(":", [for p in var.config.github.app_parameters.key_base64 : p.name]) PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME = join(":", [for p in var.config.github.app_parameters.installation_id : p != null ? p.name : ""]) - POWERTOOLS_LOGGER_LOG_EVENT = var.config.observability.logs.level == "debug" ? "true" : "false" - POWERTOOLS_METRICS_NAMESPACE = var.config.observability.metrics.namespace - POWERTOOLS_TRACE_ENABLED = var.config.observability.tracing.mode != null - POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.config.observability.tracing.capture_http_requests - POWERTOOLS_TRACER_CAPTURE_ERROR = var.config.observability.tracing.capture_error - RUNNER_LABELS = lower(join(",", var.config.runner.labels)) - RUNNER_GROUP_NAME = var.config.runner.group_name - RUNNER_NAME_PREFIX = var.config.runner.name_prefix - COMPUTE_PROVIDER_TYPE = var.runner_provider.type - RUNNERS_MAXIMUM_COUNT = var.config.runner.maximum_count - POWERTOOLS_SERVICE_NAME = "${var.config.prefix}-scale-up" SSM_TOKEN_PATH = var.config.ssm.token_path SSM_CONFIG_PATH = var.config.ssm.config_path SSM_PARAMETER_STORE_TAGS = var.config.ssm.parameter_store_tags - JOB_RETRY_CONFIG = jsonencode(local.job_retry_config) - }) + } : {}, var.storage_provider.scale_up.environment_variables) } dynamic "vpc_config" { diff --git a/modules/orchestration-providers/webhook/scale-runners/variables.tf b/modules/orchestration-providers/webhook/scale-runners/variables.tf index e191e303d1..82baab3f0a 100644 --- a/modules/orchestration-providers/webhook/scale-runners/variables.tf +++ b/modules/orchestration-providers/webhook/scale-runners/variables.tf @@ -231,3 +231,29 @@ variable "runner_provider" { nullable = false } + +variable "storage_provider" { + description = "Opaque storage-provider capabilities for scale-up and scale-down." + type = object({ + type = string + scale_up = object({ + environment_variables = map(string) + iam_policy_json = optional(string, null) + }) + scale_down = object({ + environment_variables = map(string) + iam_policy_json = optional(string, null) + }) + }) + default = { + type = "aws_ssm" + scale_up = { + environment_variables = {} + iam_policy_json = null + } + scale_down = { + environment_variables = {} + iam_policy_json = null + } + } +} diff --git a/modules/orchestration-providers/webhook/variables.tf b/modules/orchestration-providers/webhook/variables.tf index 5dfecdbd6c..78cfb41bf6 100644 --- a/modules/orchestration-providers/webhook/variables.tf +++ b/modules/orchestration-providers/webhook/variables.tf @@ -215,6 +215,48 @@ variable "ssm" { }) } +variable "storage_provider" { + description = "Opaque storage-provider environment and IAM capabilities for webhook control-plane functions." + type = object({ + type = string + scale_up = object({ + environment_variables = map(string) + iam_policy_json = optional(string, null) + }) + scale_down = object({ + environment_variables = map(string) + iam_policy_json = optional(string, null) + }) + pool = object({ + environment_variables = map(string) + iam_policy_json = optional(string, null) + }) + job_retry = object({ + environment_variables = map(string) + iam_policy_json = optional(string, null) + }) + }) + default = { + type = "aws_ssm" + scale_up = { + environment_variables = {} + iam_policy_json = null + } + scale_down = { + environment_variables = {} + iam_policy_json = null + } + pool = { + environment_variables = {} + iam_policy_json = null + } + job_retry = { + environment_variables = {} + iam_policy_json = null + } + } +} + variable "observability" { description = "Common logging, tracing, and metrics configuration consumed by webhook controls." type = object({ diff --git a/modules/storage-providers/aws/dynamodb/README.md b/modules/storage-providers/aws/dynamodb/README.md new file mode 100644 index 0000000000..9b25e0f927 --- /dev/null +++ b/modules/storage-providers/aws/dynamodb/README.md @@ -0,0 +1,59 @@ +# DynamoDB runner-config storage provider + +This internal module creates the two shared DynamoDB tables used by an opt-in multi-runner v2 deployment: one durable configuration table and one TTL-enabled runner-state table. It stores global and per-entry configuration under capability-specific partition-key scopes and returns opaque Lambda and runner capabilities with matching least-privilege IAM policies. + +The durable table isolates GitHub App credentials, webhook secrets, matcher configuration, runner-group cache entries, and bootstrap records by `scope`. The runner-state table keeps lifecycle inventory under entry-specific scopes and one-time registration configuration under the compute resource's access scope. For EC2, `compute-resource` means the full source-instance ARN; the runner can atomically read and delete only its own unexpired record. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.0 | +| [aws](#requirement\_aws) | >= 6.33 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 6.33 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_dynamodb_table.config](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dynamodb_table) | resource | +| [aws_dynamodb_table.runner_state](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dynamodb_table) | resource | +| [aws_dynamodb_table_item.github_app_credentials](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dynamodb_table_item) | resource | +| [aws_dynamodb_table_item.github_webhook_secret](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dynamodb_table_item) | resource | +| [aws_dynamodb_table_item.runner_config](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dynamodb_table_item) | resource | +| [aws_dynamodb_table_item.runner_matcher_config](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dynamodb_table_item) | resource | +| [terraform_data.config_version](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [config](#input\_config) | Settings for the shared durable configuration table and ephemeral runner-state table.

- `config.kms_key_arn`: Optional customer-managed KMS key ARN for durable configuration encryption. Null uses the AWS-owned DynamoDB key.
- `config.point_in_time_recovery_enabled`: Enables point-in-time recovery for durable configuration.
- `config.deletion_protection_enabled`: Enables deletion protection for the durable table.
- `config.tags`: Tags applied after the shared tag map.
- `runner_state.kms_key_arn`: Optional customer-managed KMS key ARN for runner-state encryption. Null uses the AWS-owned DynamoDB key.
- `runner_state.point_in_time_recovery_enabled`: Enables point-in-time recovery for ephemeral runner state.
- `runner_state.deletion_protection_enabled`: Enables deletion protection for the runner-state table.
- `runner_state.tags`: Tags applied after the shared tag map. | 
object({
    config = object({
      kms_key_arn                    = optional(string, null)
      point_in_time_recovery_enabled = optional(bool, true)
      deletion_protection_enabled    = optional(bool, false)
      tags                           = optional(map(string), {})
    })
    runner_state = object({
      kms_key_arn                    = optional(string, null)
      point_in_time_recovery_enabled = optional(bool, false)
      deletion_protection_enabled    = optional(bool, false)
      tags                           = optional(map(string), {})
    })
  })
| n/a | yes | +| [entry\_ids](#input\_entry\_ids) | Runner-entry identifiers used to build entry-scoped Lambda capabilities. | `set(string)` | n/a | yes | +| [entry\_records](#input\_entry\_records) | Resolved durable runner bootstrap configuration keyed by runner-entry identifier. | 
map(object({
    run_as                 = string
    agent_mode             = string
    disable_default_labels = bool
    enable_jit_config      = bool
  }))
| n/a | yes | +| [global\_records](#input\_global\_records) | Terraform-managed values stored under the shared global scope. | 
object({
    github_app_credentials = string
    github_webhook_secret  = string
    runner_matcher_config  = string
  })
| n/a | yes | +| [prefix](#input\_prefix) | Multi-runner prefix used to name the two shared DynamoDB tables. | `string` | n/a | yes | +| [runner\_config\_access\_scope\_prefixes](#input\_runner\_config\_access\_scope\_prefixes) | Per-entry compute-resource scope prefixes used to constrain one-time runner-config writes. | `map(string)` | n/a | yes | +| [runner\_config\_ttl\_seconds](#input\_runner\_config\_ttl\_seconds) | TTL in seconds for one-time registration and JIT configuration records. | `number` | n/a | yes | +| [runner\_state\_ttl\_seconds](#input\_runner\_state\_ttl\_seconds) | Safety TTL in seconds applied only while lifecycle records are provisioning or terminating; active and orphan inventory has no expiry. | `number` | n/a | yes | +| [tags](#input\_tags) | Base tags added to both shared DynamoDB tables. Table-specific tags override matching keys. | `map(string)` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [capabilities](#output\_capabilities) | Opaque environment and least-privilege IAM additions consumed by the shared webhook and each runner entry's control-plane functions. | +| [config\_table](#output\_config\_table) | Shared durable configuration table. Global and runner-entry records are separated by the `scope` partition key. | +| [runner\_state\_table](#output\_runner\_state\_table) | Shared TTL-backed table containing ephemeral runner configuration and provider-neutral runner lifecycle records. | + diff --git a/modules/storage-providers/aws/dynamodb/capabilities.tf b/modules/storage-providers/aws/dynamodb/capabilities.tf new file mode 100644 index 0000000000..b5f47d36ea --- /dev/null +++ b/modules/storage-providers/aws/dynamodb/capabilities.tf @@ -0,0 +1,243 @@ +locals { + config_environment_variables = { + RUNNER_CONFIG_STORAGE_PROVIDER = "aws_dynamodb" + RUNNER_CONFIG_STORAGE_VERSION = terraform_data.config_version.id + RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME = aws_dynamodb_table.config.name + } + + matcher_environment_variables = merge(local.config_environment_variables, { + RUNNER_MATCHER_CONFIG_VERSION = nonsensitive(sha256(var.global_records.runner_matcher_config)) + }) + + runner_state_environment_variables = { + RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TABLE_NAME = aws_dynamodb_table.runner_state.name + RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TTL_SECONDS = tostring(var.runner_state_ttl_seconds) + } + + runner_config_environment_variables = { + RUNNER_CONFIG_DYNAMODB_TTL_SECONDS = tostring(var.runner_config_ttl_seconds) + } + + global_scopes = { + github_app = "global#github-app" + webhook = "global#webhook" + matcher = "global#matcher" + } + + entry_scopes = { + for entry_id in var.entry_ids : entry_id => { + bootstrap = "entry#${entry_id}#bootstrap" + runner_group = "entry#${entry_id}#runner-group" + runner_state = "entry#${entry_id}#runner-state" + } + } + + entry_environment_variables = { + for entry_id, scopes in local.entry_scopes : entry_id => merge(local.config_environment_variables, { + RUNNER_CONFIG_DYNAMODB_ENTRY_ID = entry_id + }) + } + + scale_up_environment_variables = { + for entry_id in var.entry_ids : entry_id => merge( + local.entry_environment_variables[entry_id], + local.runner_state_environment_variables, + local.runner_config_environment_variables, + ) + } + + scale_down_environment_variables = { + for entry_id in var.entry_ids : entry_id => merge( + local.entry_environment_variables[entry_id], + local.runner_state_environment_variables, + ) + } + + github_app_read_statement = { + Effect = "Allow" + Action = ["dynamodb:GetItem"] + Resource = [aws_dynamodb_table.config.arn] + Condition = { + "ForAllValues:StringEquals" = { + "dynamodb:LeadingKeys" = [local.global_scopes.github_app] + } + } + } + + direct_webhook_read_statement = { + Effect = "Allow" + Action = ["dynamodb:GetItem"] + Resource = [aws_dynamodb_table.config.arn] + Condition = { + "ForAllValues:StringEquals" = { + "dynamodb:LeadingKeys" = [local.global_scopes.webhook, local.global_scopes.matcher] + } + } + } + + eventbridge_webhook_read_statement = { + Effect = "Allow" + Action = ["dynamodb:GetItem"] + Resource = [aws_dynamodb_table.config.arn] + Condition = { + "ForAllValues:StringEquals" = { + "dynamodb:LeadingKeys" = [local.global_scopes.webhook] + } + } + } + + dispatcher_read_statement = { + Effect = "Allow" + Action = ["dynamodb:GetItem"] + Resource = [aws_dynamodb_table.config.arn] + Condition = { + "ForAllValues:StringEquals" = { + "dynamodb:LeadingKeys" = [local.global_scopes.matcher] + } + } + } + + direct_webhook_iam_policy_json = jsonencode({ + Version = "2012-10-17" + Statement = [local.direct_webhook_read_statement] + }) + + eventbridge_webhook_iam_policy_json = jsonencode({ + Version = "2012-10-17" + Statement = [local.eventbridge_webhook_read_statement] + }) + + dispatcher_iam_policy_json = jsonencode({ + Version = "2012-10-17" + Statement = [local.dispatcher_read_statement] + }) + + entry_runner_group_statements = { + for entry_id, scopes in local.entry_scopes : entry_id => { + Effect = "Allow" + Action = ["dynamodb:GetItem", "dynamodb:PutItem"] + Resource = [aws_dynamodb_table.config.arn] + Condition = { + "ForAllValues:StringEquals" = { + "dynamodb:LeadingKeys" = [scopes.runner_group] + } + } + } + } + + runner_config_write_statements = { + for entry_id, scopes in local.entry_scopes : entry_id => { + Effect = "Allow" + Action = ["dynamodb:PutItem"] + Resource = [aws_dynamodb_table.runner_state.arn] + Condition = { + "ForAllValues:StringLike" = { + "dynamodb:LeadingKeys" = ["${lookup(var.runner_config_access_scope_prefixes, entry_id, "__missing_runner_config_access_scope__")}*"] + } + } + } + } + + runner_state_write_statements = { + for entry_id, scopes in local.entry_scopes : entry_id => { + Effect = "Allow" + Action = [ + "dynamodb:PutItem", + "dynamodb:Query", + "dynamodb:UpdateItem", + ] + Resource = [aws_dynamodb_table.runner_state.arn] + Condition = { + "ForAllValues:StringEquals" = { + "dynamodb:LeadingKeys" = [scopes.runner_state] + } + } + } + } + + runner_state_reconcile_statements = { + for entry_id, scopes in local.entry_scopes : entry_id => { + Effect = "Allow" + Action = [ + "dynamodb:DeleteItem", + "dynamodb:Query", + "dynamodb:UpdateItem", + ] + Resource = [aws_dynamodb_table.runner_state.arn] + Condition = { + "ForAllValues:StringEquals" = { + "dynamodb:LeadingKeys" = [scopes.runner_state] + } + } + } + } + + scale_up_iam_policy_json = { + for entry_id in var.entry_ids : entry_id => jsonencode({ + Version = "2012-10-17" + Statement = [ + local.github_app_read_statement, + local.entry_runner_group_statements[entry_id], + local.runner_config_write_statements[entry_id], + local.runner_state_write_statements[entry_id], + ] + }) + } + + scale_down_iam_policy_json = { + for entry_id in var.entry_ids : entry_id => jsonencode({ + Version = "2012-10-17" + Statement = [local.github_app_read_statement, local.runner_state_reconcile_statements[entry_id]] + }) + } + + pool_iam_policy_json = { + for entry_id in var.entry_ids : entry_id => jsonencode({ + Version = "2012-10-17" + Statement = [ + local.github_app_read_statement, + local.entry_runner_group_statements[entry_id], + local.runner_config_write_statements[entry_id], + local.runner_state_write_statements[entry_id], + ] + }) + } + + job_retry_iam_policy_json = { + for entry_id in var.entry_ids : entry_id => jsonencode({ + Version = "2012-10-17" + Statement = [local.github_app_read_statement] + }) + } + + runner_iam_policy_json = { + for entry_id, scopes in local.entry_scopes : entry_id => jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = ["dynamodb:GetItem"] + Resource = [aws_dynamodb_table.config.arn] + Condition = { + "ForAllValues:StringEquals" = { + "dynamodb:LeadingKeys" = [scopes.bootstrap] + } + } + }, + { + Effect = "Allow" + Action = [ + "dynamodb:DeleteItem", + "dynamodb:GetItem", + ] + Resource = [aws_dynamodb_table.runner_state.arn] + Condition = { + "ForAllValues:StringEquals" = { + "dynamodb:LeadingKeys" = ["$${ec2:SourceInstanceARN}"] + } + } + }, + ] + }) + } +} diff --git a/modules/storage-providers/aws/dynamodb/config-version.tf b/modules/storage-providers/aws/dynamodb/config-version.tf new file mode 100644 index 0000000000..a0bd45605c --- /dev/null +++ b/modules/storage-providers/aws/dynamodb/config-version.tf @@ -0,0 +1,23 @@ +resource "terraform_data" "config_version" { + triggers_replace = sensitive({ + global_records = sha256(jsonencode(var.global_records)) + entry_records = sha256(jsonencode(var.entry_records)) + }) + + lifecycle { + precondition { + condition = toset(keys(var.runner_config_access_scope_prefixes)) == var.entry_ids && alltrue([for prefix in values(var.runner_config_access_scope_prefixes) : trimspace(prefix) != ""]) + error_message = "runner_config_access_scope_prefixes must contain one non-empty prefix for every entry_id." + } + + precondition { + condition = var.runner_state_ttl_seconds > var.runner_config_ttl_seconds && floor(var.runner_state_ttl_seconds) == var.runner_state_ttl_seconds + error_message = "runner_state_ttl_seconds must be an integer greater than runner_config_ttl_seconds." + } + + precondition { + condition = toset(keys(var.entry_records)) == var.entry_ids + error_message = "entry_records must contain exactly one durable bootstrap record for every entry_id." + } + } +} diff --git a/modules/storage-providers/aws/dynamodb/items.tf b/modules/storage-providers/aws/dynamodb/items.tf new file mode 100644 index 0000000000..bc34f70e19 --- /dev/null +++ b/modules/storage-providers/aws/dynamodb/items.tf @@ -0,0 +1,56 @@ +resource "aws_dynamodb_table_item" "github_app_credentials" { + table_name = aws_dynamodb_table.config.name + hash_key = aws_dynamodb_table.config.hash_key + range_key = aws_dynamodb_table.config.range_key + + item = jsonencode({ + scope = { S = local.global_scopes.github_app } + id = { S = "github-app-credentials" } + value = { S = var.global_records.github_app_credentials } + }) +} + +resource "aws_dynamodb_table_item" "github_webhook_secret" { + table_name = aws_dynamodb_table.config.name + hash_key = aws_dynamodb_table.config.hash_key + range_key = aws_dynamodb_table.config.range_key + + item = jsonencode({ + scope = { S = local.global_scopes.webhook } + id = { S = "github-webhook-secret" } + value = { S = var.global_records.github_webhook_secret } + }) +} + +resource "aws_dynamodb_table_item" "runner_matcher_config" { + table_name = aws_dynamodb_table.config.name + hash_key = aws_dynamodb_table.config.hash_key + range_key = aws_dynamodb_table.config.range_key + + item = jsonencode({ + scope = { S = local.global_scopes.matcher } + id = { S = "runner-matcher-config" } + value = { S = var.global_records.runner_matcher_config } + }) +} + +resource "aws_dynamodb_table_item" "runner_config" { + for_each = var.entry_records + + table_name = aws_dynamodb_table.config.name + hash_key = aws_dynamodb_table.config.hash_key + range_key = aws_dynamodb_table.config.range_key + + item = jsonencode({ + scope = { S = local.entry_scopes[each.key].bootstrap } + id = { S = "runner-config" } + value = { S = jsonencode(merge(each.value, { + runner_config_storage = { + provider = "aws_dynamodb" + table_name = aws_dynamodb_table.runner_state.name + access_scope = "compute-resource" + id = "config" + } + })) } + }) +} diff --git a/modules/storage-providers/aws/dynamodb/outputs.tf b/modules/storage-providers/aws/dynamodb/outputs.tf new file mode 100644 index 0000000000..4c337b212d --- /dev/null +++ b/modules/storage-providers/aws/dynamodb/outputs.tf @@ -0,0 +1,71 @@ +output "config_table" { + description = "Shared durable configuration table. Global and runner-entry records are separated by the `scope` partition key." + value = { + arn = aws_dynamodb_table.config.arn + name = aws_dynamodb_table.config.name + } +} + +output "runner_state_table" { + description = "Shared TTL-backed table containing ephemeral runner configuration and provider-neutral runner lifecycle records." + value = { + arn = aws_dynamodb_table.runner_state.arn + name = aws_dynamodb_table.runner_state.name + ttl_attribute_name = "expires_at" + } +} + +output "capabilities" { + description = "Opaque environment and least-privilege IAM additions consumed by the shared webhook and each runner entry's control-plane functions." + depends_on = [ + aws_dynamodb_table_item.github_app_credentials, + aws_dynamodb_table_item.github_webhook_secret, + aws_dynamodb_table_item.runner_matcher_config, + aws_dynamodb_table_item.runner_config, + terraform_data.config_version, + ] + value = { + webhook = { + direct = { + environment_variables = tomap(local.matcher_environment_variables) + iam_policy_json = local.direct_webhook_iam_policy_json + } + eventbridge = { + webhook = { + environment_variables = tomap(local.config_environment_variables) + iam_policy_json = local.eventbridge_webhook_iam_policy_json + } + dispatcher = { + environment_variables = tomap(local.matcher_environment_variables) + iam_policy_json = local.dispatcher_iam_policy_json + } + } + } + entries = { + for entry_id in var.entry_ids : entry_id => { + scale_up = { + environment_variables = tomap(local.scale_up_environment_variables[entry_id]) + iam_policy_json = local.scale_up_iam_policy_json[entry_id] + } + scale_down = { + environment_variables = tomap(local.scale_down_environment_variables[entry_id]) + iam_policy_json = local.scale_down_iam_policy_json[entry_id] + } + pool = { + environment_variables = tomap(local.scale_up_environment_variables[entry_id]) + iam_policy_json = local.pool_iam_policy_json[entry_id] + } + job_retry = { + environment_variables = tomap(local.config_environment_variables) + iam_policy_json = local.job_retry_iam_policy_json[entry_id] + } + runner = { + config_table_name = aws_dynamodb_table.config.name + runner_state_table_name = aws_dynamodb_table.runner_state.name + scope = local.entry_scopes[entry_id].bootstrap + iam_policy_json = local.runner_iam_policy_json[entry_id] + } + } + } + } +} diff --git a/modules/storage-providers/aws/dynamodb/tables.tf b/modules/storage-providers/aws/dynamodb/tables.tf new file mode 100644 index 0000000000..12cd94867c --- /dev/null +++ b/modules/storage-providers/aws/dynamodb/tables.tf @@ -0,0 +1,62 @@ +resource "aws_dynamodb_table" "config" { + name = "${var.prefix}-config" + billing_mode = "PAY_PER_REQUEST" + hash_key = "scope" + range_key = "id" + + attribute { + name = "scope" + type = "S" + } + + attribute { + name = "id" + type = "S" + } + + point_in_time_recovery { + enabled = var.config.config.point_in_time_recovery_enabled + } + + server_side_encryption { + enabled = true + kms_key_arn = var.config.config.kms_key_arn + } + + deletion_protection_enabled = var.config.config.deletion_protection_enabled + tags = merge(var.tags, var.config.config.tags) +} + +resource "aws_dynamodb_table" "runner_state" { + name = "${var.prefix}-runner-state" + billing_mode = "PAY_PER_REQUEST" + hash_key = "scope" + range_key = "id" + + attribute { + name = "scope" + type = "S" + } + + attribute { + name = "id" + type = "S" + } + + ttl { + attribute_name = "expires_at" + enabled = true + } + + point_in_time_recovery { + enabled = var.config.runner_state.point_in_time_recovery_enabled + } + + server_side_encryption { + enabled = true + kms_key_arn = var.config.runner_state.kms_key_arn + } + + deletion_protection_enabled = var.config.runner_state.deletion_protection_enabled + tags = merge(var.tags, var.config.runner_state.tags) +} diff --git a/modules/storage-providers/aws/dynamodb/tests/provider.tftest.hcl b/modules/storage-providers/aws/dynamodb/tests/provider.tftest.hcl new file mode 100644 index 0000000000..2aac0c62b8 --- /dev/null +++ b/modules/storage-providers/aws/dynamodb/tests/provider.tftest.hcl @@ -0,0 +1,284 @@ +mock_provider "aws" { + mock_resource "aws_dynamodb_table" { + defaults = { + arn = "arn:aws:dynamodb:eu-west-1:123456789012:table/test" + } + } +} + +variables { + prefix = "github-actions" + entry_ids = ["linux", "microvm"] + runner_config_access_scope_prefixes = { + linux = "arn:aws:ec2:eu-west-1:123456789012:instance/" + microvm = "arn:aws:ec2:eu-west-1:123456789012:instance/" + } + runner_config_ttl_seconds = 3600 + runner_state_ttl_seconds = 604800 + global_records = { + github_app_credentials = jsonencode([{ appId = 123456, privateKeyBase64 = "dGVzdA==" }]) + github_webhook_secret = "test-secret" + runner_matcher_config = jsonencode([{ key = "linux" }]) + } + entry_records = { + linux = { + run_as = "runner" + agent_mode = "ephemeral" + disable_default_labels = false + enable_jit_config = true + } + microvm = { + run_as = "root" + agent_mode = "ephemeral" + disable_default_labels = true + enable_jit_config = true + } + } + tags = { + Environment = "test" + Shared = "base" + } + config = { + config = { + kms_key_arn = "arn:aws:kms:eu-west-1:123456789012:key/config" + point_in_time_recovery_enabled = true + deletion_protection_enabled = true + tags = { + Shared = "config" + } + } + runner_state = { + kms_key_arn = "arn:aws:kms:eu-west-1:123456789012:key/runner-state" + point_in_time_recovery_enabled = false + deletion_protection_enabled = false + tags = { + Shared = "runner-state" + } + } + } +} + +run "creates_two_shared_scoped_tables" { + command = apply + + assert { + condition = ( + aws_dynamodb_table.config.name == "github-actions-config" + && aws_dynamodb_table.config.billing_mode == "PAY_PER_REQUEST" + && aws_dynamodb_table.config.hash_key == "scope" + && aws_dynamodb_table.config.range_key == "id" + && aws_dynamodb_table.config.point_in_time_recovery[0].enabled + && aws_dynamodb_table.config.deletion_protection_enabled + && aws_dynamodb_table.config.server_side_encryption[0].kms_key_arn == "arn:aws:kms:eu-west-1:123456789012:key/config" + && aws_dynamodb_table.config.tags["Shared"] == "config" + ) + error_message = "The durable provider table must be one encrypted, scoped, on-demand table for the whole multi-runner deployment." + } + + assert { + condition = ( + aws_dynamodb_table.runner_state.name == "github-actions-runner-state" + && aws_dynamodb_table.runner_state.billing_mode == "PAY_PER_REQUEST" + && aws_dynamodb_table.runner_state.hash_key == "scope" + && aws_dynamodb_table.runner_state.range_key == "id" + && aws_dynamodb_table.runner_state.ttl[0].enabled + && aws_dynamodb_table.runner_state.ttl[0].attribute_name == "expires_at" + && !aws_dynamodb_table.runner_state.point_in_time_recovery[0].enabled + && !aws_dynamodb_table.runner_state.deletion_protection_enabled + && aws_dynamodb_table.runner_state.server_side_encryption[0].kms_key_arn == "arn:aws:kms:eu-west-1:123456789012:key/runner-state" + && aws_dynamodb_table.runner_state.tags["Shared"] == "runner-state" + ) + error_message = "The runner-state provider table must be one encrypted, TTL-backed, scoped, on-demand table for the whole multi-runner deployment." + } + + assert { + condition = ( + output.config_table.name == "github-actions-config" + && output.runner_state_table.name == "github-actions-runner-state" + && output.runner_state_table.ttl_attribute_name == "expires_at" + ) + error_message = "The provider outputs must expose the two shared table contracts." + } + + assert { + condition = ( + output.capabilities.webhook.direct.environment_variables["RUNNER_CONFIG_STORAGE_PROVIDER"] == "aws_dynamodb" + && output.capabilities.webhook.direct.environment_variables["RUNNER_CONFIG_DYNAMODB_CONFIG_TABLE_NAME"] == "github-actions-config" + && output.capabilities.webhook.direct.environment_variables["RUNNER_CONFIG_STORAGE_VERSION"] == terraform_data.config_version.id + && output.capabilities.webhook.eventbridge.webhook.environment_variables["RUNNER_CONFIG_STORAGE_VERSION"] == terraform_data.config_version.id + && output.capabilities.webhook.eventbridge.dispatcher.environment_variables["RUNNER_CONFIG_STORAGE_VERSION"] == terraform_data.config_version.id + && output.capabilities.webhook.direct.environment_variables["RUNNER_MATCHER_CONFIG_VERSION"] == sha256(jsonencode([{ key = "linux" }])) + && !contains(keys(output.capabilities.webhook.direct.environment_variables), "RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TABLE_NAME") + && !contains(keys(output.capabilities.webhook.direct.environment_variables), "RUNNER_CONFIG_DYNAMODB_TTL_SECONDS") + && !contains(keys(output.capabilities.webhook.eventbridge.webhook.environment_variables), "RUNNER_MATCHER_CONFIG_VERSION") + && output.capabilities.webhook.eventbridge.dispatcher.environment_variables["RUNNER_MATCHER_CONFIG_VERSION"] == sha256(jsonencode([{ key = "linux" }])) + && output.capabilities.entries["linux"].scale_up.environment_variables["RUNNER_CONFIG_DYNAMODB_ENTRY_ID"] == "linux" + && output.capabilities.entries["linux"].scale_up.environment_variables["RUNNER_CONFIG_STORAGE_VERSION"] == terraform_data.config_version.id + && !contains(keys(output.capabilities.entries["linux"].scale_up.environment_variables), "RUNNER_MATCHER_CONFIG_VERSION") + && output.capabilities.entries["microvm"].scale_down.environment_variables["RUNNER_CONFIG_DYNAMODB_ENTRY_ID"] == "microvm" + && output.capabilities.entries["microvm"].scale_down.environment_variables["RUNNER_CONFIG_STORAGE_VERSION"] == terraform_data.config_version.id + && output.capabilities.entries["linux"].pool.environment_variables["RUNNER_CONFIG_DYNAMODB_TTL_SECONDS"] == "3600" + && output.capabilities.entries["linux"].scale_down.environment_variables["RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TTL_SECONDS"] == "604800" + && !contains(keys(output.capabilities.entries["linux"].scale_down.environment_variables), "RUNNER_CONFIG_DYNAMODB_TTL_SECONDS") + && !contains(keys(output.capabilities.entries["linux"].job_retry.environment_variables), "RUNNER_CONFIG_DYNAMODB_ENTRY_ID") + && !contains(keys(output.capabilities.entries["linux"].job_retry.environment_variables), "RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TABLE_NAME") + && output.capabilities.entries["linux"].job_retry.environment_variables["RUNNER_CONFIG_STORAGE_VERSION"] == terraform_data.config_version.id + && output.capabilities.entries["linux"].runner.config_table_name == "github-actions-config" + && output.capabilities.entries["linux"].runner.runner_state_table_name == "github-actions-runner-state" + && output.capabilities.entries["linux"].runner.scope == "entry#linux#bootstrap" + ) + error_message = "The provider must expose one global and entry-scoped Lambda environment contract over the same two tables." + } + + assert { + condition = ( + jsondecode(output.capabilities.webhook.direct.iam_policy_json).Statement[0].Condition["ForAllValues:StringEquals"]["dynamodb:LeadingKeys"] == ["global#webhook", "global#matcher"] + && jsondecode(output.capabilities.webhook.eventbridge.webhook.iam_policy_json).Statement[0].Condition["ForAllValues:StringEquals"]["dynamodb:LeadingKeys"] == ["global#webhook"] + && jsondecode(output.capabilities.webhook.eventbridge.dispatcher.iam_policy_json).Statement[0].Condition["ForAllValues:StringEquals"]["dynamodb:LeadingKeys"] == ["global#matcher"] + && jsondecode(output.capabilities.entries["linux"].scale_up.iam_policy_json).Statement[0].Action == ["dynamodb:GetItem"] + && jsondecode(output.capabilities.entries["linux"].scale_up.iam_policy_json).Statement[0].Condition["ForAllValues:StringEquals"]["dynamodb:LeadingKeys"] == ["global#github-app"] + && jsondecode(output.capabilities.entries["linux"].scale_up.iam_policy_json).Statement[1].Action == ["dynamodb:GetItem", "dynamodb:PutItem"] + && jsondecode(output.capabilities.entries["linux"].scale_up.iam_policy_json).Statement[1].Condition["ForAllValues:StringEquals"]["dynamodb:LeadingKeys"] == ["entry#linux#runner-group"] + && jsondecode(output.capabilities.entries["linux"].scale_up.iam_policy_json).Statement[2].Action == ["dynamodb:PutItem"] + && jsondecode(output.capabilities.entries["linux"].scale_up.iam_policy_json).Statement[2].Condition["ForAllValues:StringLike"]["dynamodb:LeadingKeys"] == ["arn:aws:ec2:eu-west-1:123456789012:instance/*"] + && jsondecode(output.capabilities.entries["linux"].scale_up.iam_policy_json).Statement[3].Action == ["dynamodb:PutItem", "dynamodb:Query", "dynamodb:UpdateItem"] + && jsondecode(output.capabilities.entries["linux"].scale_up.iam_policy_json).Statement[3].Condition["ForAllValues:StringEquals"]["dynamodb:LeadingKeys"] == ["entry#linux#runner-state"] + && jsondecode(output.capabilities.entries["microvm"].scale_down.iam_policy_json).Statement[1].Action == ["dynamodb:DeleteItem", "dynamodb:Query", "dynamodb:UpdateItem"] + && jsondecode(output.capabilities.entries["microvm"].scale_down.iam_policy_json).Statement[1].Condition["ForAllValues:StringEquals"]["dynamodb:LeadingKeys"] == ["entry#microvm#runner-state"] + && !contains(jsondecode(output.capabilities.entries["linux"].pool.iam_policy_json).Statement[3].Action, "dynamodb:DeleteItem") + && jsondecode(output.capabilities.entries["linux"].runner.iam_policy_json).Statement[0].Condition["ForAllValues:StringEquals"]["dynamodb:LeadingKeys"] == ["entry#linux#bootstrap"] + && jsondecode(output.capabilities.entries["linux"].runner.iam_policy_json).Statement[1].Condition["ForAllValues:StringEquals"]["dynamodb:LeadingKeys"] == ["$${ec2:SourceInstanceARN}"] + ) + error_message = "Provider IAM capabilities must restrict global and entry operations with DynamoDB leading-key conditions." + } + + + assert { + condition = ( + jsondecode(aws_dynamodb_table_item.github_app_credentials.item).scope.S == "global#github-app" + && jsondecode(aws_dynamodb_table_item.github_webhook_secret.item).scope.S == "global#webhook" + && jsondecode(aws_dynamodb_table_item.runner_matcher_config.item).scope.S == "global#matcher" + && jsondecode(aws_dynamodb_table_item.runner_config["linux"].item).scope.S == "entry#linux#bootstrap" + && jsondecode(aws_dynamodb_table_item.runner_config["linux"].item).id.S == "runner-config" + && jsondecode(jsondecode(aws_dynamodb_table_item.runner_config["linux"].item).value.S).run_as == "runner" + && jsondecode(jsondecode(aws_dynamodb_table_item.runner_config["linux"].item).value.S).runner_config_storage.table_name == "github-actions-runner-state" + && jsondecode(jsondecode(aws_dynamodb_table_item.runner_config["linux"].item).value.S).runner_config_storage.access_scope == "compute-resource" + && jsondecode(jsondecode(aws_dynamodb_table_item.runner_config["linux"].item).value.S).runner_config_storage.id == "config" + ) + error_message = "Each entry must receive one durable bootstrap record that points at its scope in the shared runner-state table." + } +} + +run "storage_version_tracks_global_record_changes" { + command = apply + + variables { + global_records = { + github_app_credentials = jsonencode([{ appId = 123456, privateKeyBase64 = "dGVzdA==" }]) + github_webhook_secret = "rotated-test-secret" + runner_matcher_config = jsonencode([{ key = "linux" }]) + } + } + + assert { + condition = ( + output.capabilities.webhook.direct.environment_variables["RUNNER_CONFIG_STORAGE_VERSION"] == terraform_data.config_version.id + && output.capabilities.entries["linux"].job_retry.environment_variables["RUNNER_CONFIG_STORAGE_VERSION"] == terraform_data.config_version.id + ) + error_message = "A durable global-record update must publish the replacement storage resource ID to every Lambda capability." + } +} + +run "matcher_version_tracks_matcher_content" { + command = apply + + variables { + global_records = { + github_app_credentials = jsonencode([{ appId = 123456, privateKeyBase64 = "dGVzdA==" }]) + github_webhook_secret = "rotated-test-secret" + runner_matcher_config = jsonencode([{ key = "microvm" }]) + } + } + + assert { + condition = ( + output.capabilities.webhook.direct.environment_variables["RUNNER_MATCHER_CONFIG_VERSION"] == sha256(jsonencode([{ key = "microvm" }])) + && output.capabilities.webhook.direct.environment_variables["RUNNER_MATCHER_CONFIG_VERSION"] != sha256(jsonencode([{ key = "linux" }])) + && output.capabilities.webhook.direct.environment_variables["RUNNER_CONFIG_STORAGE_VERSION"] == terraform_data.config_version.id + ) + error_message = "The matcher and opaque storage versions must change without exposing the matcher payload whenever the durable matcher record changes." + } +} + +run "storage_version_tracks_entry_record_changes" { + command = apply + + variables { + global_records = { + github_app_credentials = jsonencode([{ appId = 123456, privateKeyBase64 = "dGVzdA==" }]) + github_webhook_secret = "rotated-test-secret" + runner_matcher_config = jsonencode([{ key = "microvm" }]) + } + entry_records = { + linux = { + run_as = "root" + agent_mode = "ephemeral" + disable_default_labels = false + enable_jit_config = true + } + microvm = { + run_as = "root" + agent_mode = "ephemeral" + disable_default_labels = true + enable_jit_config = true + } + } + } + + assert { + condition = ( + output.capabilities.entries["linux"].scale_up.environment_variables["RUNNER_CONFIG_STORAGE_VERSION"] == terraform_data.config_version.id + && output.capabilities.webhook.eventbridge.webhook.environment_variables["RUNNER_CONFIG_STORAGE_VERSION"] == terraform_data.config_version.id + ) + error_message = "A durable entry-record update must publish the replacement storage resource ID to every Lambda capability." + } +} + +run "rejects_missing_runner_config_access_scope_prefix" { + command = plan + + variables { + runner_config_access_scope_prefixes = { + linux = "arn:aws:ec2:eu-west-1:123456789012:instance/" + } + } + + expect_failures = [terraform_data.config_version] +} + +run "rejects_runner_state_ttl_not_greater_than_runner_config_ttl" { + command = plan + + variables { + runner_state_ttl_seconds = 3600 + } + + expect_failures = [terraform_data.config_version] +} + +run "rejects_missing_entry_record" { + command = plan + + variables { + entry_records = { + linux = { + run_as = "runner" + agent_mode = "ephemeral" + disable_default_labels = false + enable_jit_config = true + } + } + } + + expect_failures = [terraform_data.config_version] +} diff --git a/modules/storage-providers/aws/dynamodb/variables.tf b/modules/storage-providers/aws/dynamodb/variables.tf new file mode 100644 index 0000000000..dc4e90badc --- /dev/null +++ b/modules/storage-providers/aws/dynamodb/variables.tf @@ -0,0 +1,84 @@ +variable "prefix" { + description = "Multi-runner prefix used to name the two shared DynamoDB tables." + type = string +} + +variable "tags" { + description = "Base tags added to both shared DynamoDB tables. Table-specific tags override matching keys." + type = map(string) + default = {} +} + +variable "entry_ids" { + description = "Runner-entry identifiers used to build entry-scoped Lambda capabilities." + type = set(string) +} + +variable "runner_config_access_scope_prefixes" { + description = "Per-entry compute-resource scope prefixes used to constrain one-time runner-config writes." + type = map(string) +} + +variable "runner_config_ttl_seconds" { + description = "TTL in seconds for one-time registration and JIT configuration records." + type = number + + validation { + condition = var.runner_config_ttl_seconds > 0 && floor(var.runner_config_ttl_seconds) == var.runner_config_ttl_seconds + error_message = "runner_config_ttl_seconds must be a positive integer." + } +} + +variable "runner_state_ttl_seconds" { + description = "Safety TTL in seconds applied only while lifecycle records are provisioning or terminating; active and orphan inventory has no expiry." + type = number +} + +variable "global_records" { + description = "Terraform-managed values stored under the shared global scope." + type = object({ + github_app_credentials = string + github_webhook_secret = string + runner_matcher_config = string + }) + sensitive = true +} + +variable "entry_records" { + description = "Resolved durable runner bootstrap configuration keyed by runner-entry identifier." + type = map(object({ + run_as = string + agent_mode = string + disable_default_labels = bool + enable_jit_config = bool + })) +} + +variable "config" { + description = <<-EOT + Settings for the shared durable configuration table and ephemeral runner-state table. + + - `config.kms_key_arn`: Optional customer-managed KMS key ARN for durable configuration encryption. Null uses the AWS-owned DynamoDB key. + - `config.point_in_time_recovery_enabled`: Enables point-in-time recovery for durable configuration. + - `config.deletion_protection_enabled`: Enables deletion protection for the durable table. + - `config.tags`: Tags applied after the shared tag map. + - `runner_state.kms_key_arn`: Optional customer-managed KMS key ARN for runner-state encryption. Null uses the AWS-owned DynamoDB key. + - `runner_state.point_in_time_recovery_enabled`: Enables point-in-time recovery for ephemeral runner state. + - `runner_state.deletion_protection_enabled`: Enables deletion protection for the runner-state table. + - `runner_state.tags`: Tags applied after the shared tag map. + EOT + type = object({ + config = object({ + kms_key_arn = optional(string, null) + point_in_time_recovery_enabled = optional(bool, true) + deletion_protection_enabled = optional(bool, false) + tags = optional(map(string), {}) + }) + runner_state = object({ + kms_key_arn = optional(string, null) + point_in_time_recovery_enabled = optional(bool, false) + deletion_protection_enabled = optional(bool, false) + tags = optional(map(string), {}) + }) + }) +} diff --git a/modules/storage-providers/aws/dynamodb/versions.tf b/modules/storage-providers/aws/dynamodb/versions.tf new file mode 100644 index 0000000000..3ef011ea0a --- /dev/null +++ b/modules/storage-providers/aws/dynamodb/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.4.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.33" + } + } +} diff --git a/modules/webhook/README.md b/modules/webhook/README.md index 70121458a7..7c787729bb 100644 --- a/modules/webhook/README.md +++ b/modules/webhook/README.md @@ -91,6 +91,7 @@ yarn run dist | [role\_permissions\_boundary](#input\_role\_permissions\_boundary) | Permissions boundary that will be added to the created role for the lambda. | `string` | `null` | no | | [runner\_matcher\_config](#input\_runner\_matcher\_config) | SQS queue to publish accepted build events based on the runner type. `computeProvider` defaults to `ec2`; EC2 is the only provider currently implemented. When exact match is disabled the webhook accepts the event if one of the workflow job labels is part of the matcher. The priority defines the order the matchers are applied. Optional `matcherConfig.enableDynamicLabels` and `matcherConfig.awsDynamicLabelsPolicy` are evaluated by the dispatcher to gate provider dynamic labels per runner. The policy supports `blocked_keys = []` and `restricted_keys = { = { allowed = [globs], denied = [globs], max = number|string } }`; keys use the provider dynamic label suffix form, for example `instance-type` for `ghr-ec2-instance-type`. | 
map(object({
    arn             = string
    id              = string
    computeProvider = optional(string, "ec2")
    matcherConfig = object({
      labelMatchers           = list(list(string))
      exactMatch              = bool
      bidirectionalLabelMatch = optional(bool, false)
      priority                = optional(number, 999)
      enableDynamicLabels     = optional(bool, false)
      awsDynamicLabelsPolicy  = optional(any, null)
    })
  }))
| n/a | yes | | [ssm\_paths](#input\_ssm\_paths) | The root path used in SSM to store configuration and secrets. | 
object({
    root    = string
    webhook = string
  })
| n/a | yes | +| [storage\_provider](#input\_storage\_provider) | Selected storage-provider type and opaque capabilities used by the webhook and optional dispatcher Lambdas. | 
object({
    type = optional(string, "aws_ssm")
    direct = object({
      environment_variables = map(string)
      iam_policy_json       = optional(string, null)
    })
    eventbridge = object({
      webhook = object({
        environment_variables = map(string)
        iam_policy_json       = optional(string, null)
      })
      dispatcher = object({
        environment_variables = map(string)
        iam_policy_json       = optional(string, null)
      })
    })
  })
| 
{
  "direct": {
    "environment_variables": {},
    "iam_policy_json": null
  },
  "eventbridge": {
    "dispatcher": {
      "environment_variables": {},
      "iam_policy_json": null
    },
    "webhook": {
      "environment_variables": {},
      "iam_policy_json": null
    }
  },
  "type": "aws_ssm"
}
| no | | [tags](#input\_tags) | Map of tags that will be added to created resources. By default resources will be tagged with name and environment. | `map(string)` | `{}` | no | | [tracing\_config](#input\_tracing\_config) | Configuration for lambda tracing. | 
object({
    mode                  = optional(string, null)
    capture_http_requests = optional(bool, false)
    capture_error         = optional(bool, false)
  })
| `{}` | no | | [webhook\_lambda\_apigateway\_access\_log\_settings](#input\_webhook\_lambda\_apigateway\_access\_log\_settings) | Access log settings for webhook API gateway. | 
object({
    destination_arn = string
    format          = string
  })
| `null` | no | diff --git a/modules/webhook/direct/README.md b/modules/webhook/direct/README.md index d639ed6398..0dd69652aa 100644 --- a/modules/webhook/direct/README.md +++ b/modules/webhook/direct/README.md @@ -40,7 +40,7 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [config](#input\_config) | Configuration object for all variables. | 
object({
    prefix = string
    archive = optional(object({
      enable         = optional(bool, true)
      retention_days = optional(number, 7)
    }), {})
    tags = optional(map(string), {})

    lambda_subnet_ids         = optional(list(string), [])
    lambda_security_group_ids = optional(list(string), [])
    sqs_job_queues_arns       = list(string)
    lambda_zip                = optional(string, null)
    lambda_memory_size        = optional(number, 256)
    lambda_timeout            = optional(number, 10)
    role_permissions_boundary = optional(string, null)
    role_path                 = optional(string, null)
    logging_retention_in_days = optional(number, 180)
    logging_kms_key_id        = optional(string, null)
    log_class                 = optional(string, "STANDARD")
    lambda_s3_bucket          = optional(string, null)
    lambda_s3_key             = optional(string, null)
    lambda_s3_object_version  = optional(string, null)
    lambda_apigateway_access_log_settings = optional(object({
      destination_arn = string
      format          = string
    }), null)
    repository_white_list    = optional(list(string), [])
    queue_selection_strategy = optional(string, "first")
    kms_key_arn              = optional(string, null)
    log_level                = optional(string, "info")
    lambda_runtime           = optional(string, "nodejs24.x")
    aws_partition            = optional(string, "aws")
    lambda_architecture      = optional(string, "arm64")
    github_app_parameters = object({
      webhook_secret = map(string)
    })
    tracing_config = optional(object({
      mode                  = optional(string, null)
      capture_http_requests = optional(bool, false)
      capture_error         = optional(bool, false)
    }), {})
    lambda_tags       = optional(map(string), {})
    api_gw_source_arn = string
    ssm_parameter_runner_matcher_config = list(object({
      name    = string
      arn     = string
      version = string
    }))
  })
| n/a | yes | +| [config](#input\_config) | Configuration object for all variables. | 
object({
    prefix = string
    archive = optional(object({
      enable         = optional(bool, true)
      retention_days = optional(number, 7)
    }), {})
    tags = optional(map(string), {})

    lambda_subnet_ids         = optional(list(string), [])
    lambda_security_group_ids = optional(list(string), [])
    sqs_job_queues_arns       = list(string)
    lambda_zip                = optional(string, null)
    lambda_memory_size        = optional(number, 256)
    lambda_timeout            = optional(number, 10)
    role_permissions_boundary = optional(string, null)
    role_path                 = optional(string, null)
    logging_retention_in_days = optional(number, 180)
    logging_kms_key_id        = optional(string, null)
    log_class                 = optional(string, "STANDARD")
    lambda_s3_bucket          = optional(string, null)
    lambda_s3_key             = optional(string, null)
    lambda_s3_object_version  = optional(string, null)
    lambda_apigateway_access_log_settings = optional(object({
      destination_arn = string
      format          = string
    }), null)
    repository_white_list    = optional(list(string), [])
    queue_selection_strategy = optional(string, "first")
    kms_key_arn              = optional(string, null)
    log_level                = optional(string, "info")
    lambda_runtime           = optional(string, "nodejs24.x")
    aws_partition            = optional(string, "aws")
    lambda_architecture      = optional(string, "arm64")
    github_app_parameters = object({
      webhook_secret = map(string)
    })
    tracing_config = optional(object({
      mode                  = optional(string, null)
      capture_http_requests = optional(bool, false)
      capture_error         = optional(bool, false)
    }), {})
    lambda_tags       = optional(map(string), {})
    api_gw_source_arn = string
    ssm_parameter_runner_matcher_config = list(object({
      name    = string
      arn     = string
      version = string
    }))
    storage_provider = optional(object({
      type                  = optional(string, "aws_ssm")
      environment_variables = map(string)
      iam_policy_json       = optional(string, null)
      }), {
      type                  = "aws_ssm"
      environment_variables = {}
      iam_policy_json       = null
    })
  })
| n/a | yes | ## Outputs diff --git a/modules/webhook/direct/variables.tf b/modules/webhook/direct/variables.tf index 402ac514b4..1729f97b23 100644 --- a/modules/webhook/direct/variables.tf +++ b/modules/webhook/direct/variables.tf @@ -48,5 +48,14 @@ variable "config" { arn = string version = string })) + storage_provider = optional(object({ + type = optional(string, "aws_ssm") + environment_variables = map(string) + iam_policy_json = optional(string, null) + }), { + type = "aws_ssm" + environment_variables = {} + iam_policy_json = null + }) }) } diff --git a/modules/webhook/direct/webhook.tf b/modules/webhook/direct/webhook.tf index 5ef2e1ebfb..a322c3e432 100644 --- a/modules/webhook/direct/webhook.tf +++ b/modules/webhook/direct/webhook.tf @@ -19,20 +19,20 @@ resource "aws_lambda_function" "webhook" { depends_on = [aws_cloudwatch_log_group.webhook] environment { - variables = { + variables = merge({ for k, v in { LOG_LEVEL = upper(var.config.log_level) POWERTOOLS_LOGGER_LOG_EVENT = var.config.log_level == "debug" ? "true" : "false" POWERTOOLS_TRACE_ENABLED = var.config.tracing_config.mode != null ? true : false POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.config.tracing_config.capture_http_requests POWERTOOLS_TRACER_CAPTURE_ERROR = var.config.tracing_config.capture_error - PARAMETER_GITHUB_APP_WEBHOOK_SECRET = var.config.github_app_parameters.webhook_secret.name + PARAMETER_GITHUB_APP_WEBHOOK_SECRET = var.config.storage_provider.type == "aws_ssm" ? var.config.github_app_parameters.webhook_secret.name : null REPOSITORY_ALLOW_LIST = jsonencode(var.config.repository_white_list) QUEUE_SELECTION_STRATEGY = var.config.queue_selection_strategy - PARAMETER_RUNNER_MATCHER_CONFIG_PATH = join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.name]) - PARAMETER_RUNNER_MATCHER_VERSION = join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.version]) # enforce cold start after Changes in SSM parameter + PARAMETER_RUNNER_MATCHER_CONFIG_PATH = var.config.storage_provider.type == "aws_ssm" ? join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.name]) : null + PARAMETER_RUNNER_MATCHER_VERSION = var.config.storage_provider.type == "aws_ssm" ? join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.version]) : null # enforce cold start after Changes in SSM parameter } : k => v if v != null - } + }, var.config.storage_provider.environment_variables) } dynamic "vpc_config" { @@ -125,6 +125,8 @@ resource "aws_iam_role_policy" "webhook_sqs" { } resource "aws_iam_role_policy" "webhook_kms" { + count = var.config.storage_provider.type == "aws_ssm" ? 1 : 0 + name = "kms-policy" role = aws_iam_role.webhook_lambda.name @@ -133,18 +135,23 @@ resource "aws_iam_role_policy" "webhook_kms" { }) } +moved { + from = aws_iam_role_policy.webhook_kms + to = aws_iam_role_policy.webhook_kms[0] +} + resource "aws_iam_role_policy" "webhook_ssm" { name = "publish-ssm-policy" role = aws_iam_role.webhook_lambda.name - policy = templatefile("${path.module}/../policies/lambda-ssm.json", { + policy = var.config.storage_provider.type == "aws_ssm" ? templatefile("${path.module}/../policies/lambda-ssm.json", { resource_arns = jsonencode( concat( [var.config.github_app_parameters.webhook_secret.arn], [for p in var.config.ssm_parameter_runner_matcher_config : p.arn] ) ) - }) + }) : var.config.storage_provider.iam_policy_json } resource "aws_iam_role_policy" "xray" { diff --git a/modules/webhook/eventbridge/README.md b/modules/webhook/eventbridge/README.md index 07aa0bdd61..0cf90dda3f 100644 --- a/modules/webhook/eventbridge/README.md +++ b/modules/webhook/eventbridge/README.md @@ -54,7 +54,7 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [config](#input\_config) | Configuration object for all variables. | 
object({
    prefix = string
    archive = optional(object({
      enable         = optional(bool, true)
      retention_days = optional(number, 7)
    }), {})
    tags = optional(map(string), {})

    lambda_subnet_ids         = optional(list(string), [])
    lambda_security_group_ids = optional(list(string), [])
    sqs_job_queues_arns       = list(string)
    lambda_zip                = optional(string, null)
    lambda_memory_size        = optional(number, 256)
    lambda_timeout            = optional(number, 10)
    role_permissions_boundary = optional(string, null)
    role_path                 = optional(string, null)
    logging_retention_in_days = optional(number, 180)
    logging_kms_key_id        = optional(string, null)
    log_class                 = optional(string, "STANDARD")
    lambda_s3_bucket          = optional(string, null)
    lambda_s3_key             = optional(string, null)
    lambda_s3_object_version  = optional(string, null)
    lambda_apigateway_access_log_settings = optional(object({
      destination_arn = string
      format          = string
    }), null)
    repository_white_list    = optional(list(string), [])
    queue_selection_strategy = optional(string, "first")
    kms_key_arn              = optional(string, null)
    log_level                = optional(string, "info")
    lambda_runtime           = optional(string, "nodejs24.x")
    aws_partition            = optional(string, "aws")
    lambda_architecture      = optional(string, "arm64")
    github_app_parameters = object({
      webhook_secret = map(string)
    })
    tracing_config = optional(object({
      mode                  = optional(string, null)
      capture_http_requests = optional(bool, false)
      capture_error         = optional(bool, false)
    }), {})
    lambda_tags       = optional(map(string), {})
    api_gw_source_arn = string
    ssm_parameter_runner_matcher_config = list(object({
      name    = string
      arn     = string
      version = string
    }))
    accept_events = optional(list(string), null)
  })
| n/a | yes | +| [config](#input\_config) | Configuration object for all variables. | 
object({
    prefix = string
    archive = optional(object({
      enable         = optional(bool, true)
      retention_days = optional(number, 7)
    }), {})
    tags = optional(map(string), {})

    lambda_subnet_ids         = optional(list(string), [])
    lambda_security_group_ids = optional(list(string), [])
    sqs_job_queues_arns       = list(string)
    lambda_zip                = optional(string, null)
    lambda_memory_size        = optional(number, 256)
    lambda_timeout            = optional(number, 10)
    role_permissions_boundary = optional(string, null)
    role_path                 = optional(string, null)
    logging_retention_in_days = optional(number, 180)
    logging_kms_key_id        = optional(string, null)
    log_class                 = optional(string, "STANDARD")
    lambda_s3_bucket          = optional(string, null)
    lambda_s3_key             = optional(string, null)
    lambda_s3_object_version  = optional(string, null)
    lambda_apigateway_access_log_settings = optional(object({
      destination_arn = string
      format          = string
    }), null)
    repository_white_list    = optional(list(string), [])
    queue_selection_strategy = optional(string, "first")
    kms_key_arn              = optional(string, null)
    log_level                = optional(string, "info")
    lambda_runtime           = optional(string, "nodejs24.x")
    aws_partition            = optional(string, "aws")
    lambda_architecture      = optional(string, "arm64")
    github_app_parameters = object({
      webhook_secret = map(string)
    })
    tracing_config = optional(object({
      mode                  = optional(string, null)
      capture_http_requests = optional(bool, false)
      capture_error         = optional(bool, false)
    }), {})
    lambda_tags       = optional(map(string), {})
    api_gw_source_arn = string
    ssm_parameter_runner_matcher_config = list(object({
      name    = string
      arn     = string
      version = string
    }))
    storage_provider = optional(object({
      type = optional(string, "aws_ssm")
      webhook = object({
        environment_variables = map(string)
        iam_policy_json       = optional(string, null)
      })
      dispatcher = object({
        environment_variables = map(string)
        iam_policy_json       = optional(string, null)
      })
      }), {
      type = "aws_ssm"
      webhook = {
        environment_variables = {}
        iam_policy_json       = null
      }
      dispatcher = {
        environment_variables = {}
        iam_policy_json       = null
      }
    })
    accept_events = optional(list(string), null)
  })
| n/a | yes | ## Outputs diff --git a/modules/webhook/eventbridge/dispatcher.tf b/modules/webhook/eventbridge/dispatcher.tf index 39b65a6d48..3e50f9bafb 100644 --- a/modules/webhook/eventbridge/dispatcher.tf +++ b/modules/webhook/eventbridge/dispatcher.tf @@ -40,7 +40,7 @@ resource "aws_lambda_function" "dispatcher" { depends_on = [aws_cloudwatch_log_group.dispatcher] environment { - variables = { + variables = merge({ for k, v in { LOG_LEVEL = upper(var.config.log_level) POWERTOOLS_LOGGER_LOG_EVENT = var.config.log_level == "debug" ? "true" : "false" @@ -49,12 +49,12 @@ resource "aws_lambda_function" "dispatcher" { POWERTOOLS_TRACER_CAPTURE_HTTPS_REQUESTS = var.config.tracing_config.capture_http_requests POWERTOOLS_TRACER_CAPTURE_ERROR = var.config.tracing_config.capture_error # Parameters required for lambda configuration - PARAMETER_RUNNER_MATCHER_CONFIG_PATH = join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.name]) - PARAMETER_RUNNER_MATCHER_VERSION = join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.version]) # enforce cold start after Changes in SSM parameter + PARAMETER_RUNNER_MATCHER_CONFIG_PATH = var.config.storage_provider.type == "aws_ssm" ? join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.name]) : null + PARAMETER_RUNNER_MATCHER_VERSION = var.config.storage_provider.type == "aws_ssm" ? join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.version]) : null # enforce cold start after Changes in SSM parameter REPOSITORY_ALLOW_LIST = jsonencode(var.config.repository_white_list) QUEUE_SELECTION_STRATEGY = var.config.queue_selection_strategy } : k => v if v != null - } + }, var.config.storage_provider.dispatcher.environment_variables) } dynamic "vpc_config" { @@ -123,6 +123,8 @@ resource "aws_iam_role_policy" "dispatcher_sqs" { } resource "aws_iam_role_policy" "dispatcher_kms" { + count = var.config.storage_provider.type == "aws_ssm" ? 1 : 0 + name = "kms-policy" role = aws_iam_role.dispatcher_lambda.name @@ -131,17 +133,22 @@ resource "aws_iam_role_policy" "dispatcher_kms" { }) } +moved { + from = aws_iam_role_policy.dispatcher_kms + to = aws_iam_role_policy.dispatcher_kms[0] +} + resource "aws_iam_role_policy" "dispatcher_ssm" { name = "publish-ssm-policy" role = aws_iam_role.dispatcher_lambda.name - policy = templatefile("${path.module}/../policies/lambda-ssm.json", { + policy = var.config.storage_provider.type == "aws_ssm" ? templatefile("${path.module}/../policies/lambda-ssm.json", { resource_arns = jsonencode( concat( [for p in var.config.ssm_parameter_runner_matcher_config : p.arn] ) ) - }) + }) : var.config.storage_provider.dispatcher.iam_policy_json } resource "aws_iam_role_policy" "dispatcher_xray" { diff --git a/modules/webhook/eventbridge/variables.tf b/modules/webhook/eventbridge/variables.tf index c6d35d82d3..91bbd39482 100644 --- a/modules/webhook/eventbridge/variables.tf +++ b/modules/webhook/eventbridge/variables.tf @@ -48,6 +48,27 @@ variable "config" { arn = string version = string })) + storage_provider = optional(object({ + type = optional(string, "aws_ssm") + webhook = object({ + environment_variables = map(string) + iam_policy_json = optional(string, null) + }) + dispatcher = object({ + environment_variables = map(string) + iam_policy_json = optional(string, null) + }) + }), { + type = "aws_ssm" + webhook = { + environment_variables = {} + iam_policy_json = null + } + dispatcher = { + environment_variables = {} + iam_policy_json = null + } + }) accept_events = optional(list(string), null) }) } diff --git a/modules/webhook/eventbridge/webhook.tf b/modules/webhook/eventbridge/webhook.tf index 9af03279a2..9d6a01561f 100644 --- a/modules/webhook/eventbridge/webhook.tf +++ b/modules/webhook/eventbridge/webhook.tf @@ -24,7 +24,7 @@ resource "aws_lambda_function" "webhook" { depends_on = [aws_cloudwatch_log_group.webhook] environment { - variables = { + variables = merge({ for k, v in { LOG_LEVEL = upper(var.config.log_level) POWERTOOLS_LOGGER_LOG_EVENT = var.config.log_level == "debug" ? "true" : "false" @@ -35,10 +35,10 @@ resource "aws_lambda_function" "webhook" { # Parameters required for lambda configuration ACCEPT_EVENTS = jsonencode(var.config.accept_events) EVENT_BUS_NAME = aws_cloudwatch_event_bus.main.name - PARAMETER_GITHUB_APP_WEBHOOK_SECRET = var.config.github_app_parameters.webhook_secret.name - PARAMETER_RUNNER_MATCHER_CONFIG_PATH = join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.name]) + PARAMETER_GITHUB_APP_WEBHOOK_SECRET = var.config.storage_provider.type == "aws_ssm" ? var.config.github_app_parameters.webhook_secret.name : null + PARAMETER_RUNNER_MATCHER_CONFIG_PATH = var.config.storage_provider.type == "aws_ssm" ? join(":", [for p in var.config.ssm_parameter_runner_matcher_config : p.name]) : null } : k => v if v != null - } + }, var.config.storage_provider.webhook.environment_variables) } dynamic "vpc_config" { @@ -129,12 +129,14 @@ resource "aws_iam_role_policy" "webhook_ssm" { name = "publish-ssm-policy" role = aws_iam_role.webhook_lambda.name - policy = templatefile("${path.module}/../policies/lambda-ssm.json", { + policy = var.config.storage_provider.type == "aws_ssm" ? templatefile("${path.module}/../policies/lambda-ssm.json", { resource_arns = jsonencode([var.config.github_app_parameters.webhook_secret.arn]) - }) + }) : var.config.storage_provider.webhook.iam_policy_json } resource "aws_iam_role_policy" "webhook_kms" { + count = var.config.storage_provider.type == "aws_ssm" ? 1 : 0 + name = "kms-policy" role = aws_iam_role.webhook_lambda.name @@ -143,6 +145,11 @@ resource "aws_iam_role_policy" "webhook_kms" { }) } +moved { + from = aws_iam_role_policy.webhook_kms + to = aws_iam_role_policy.webhook_kms[0] +} + resource "aws_iam_role_policy" "xray" { count = var.config.tracing_config.mode != null ? 1 : 0 name = "xray-policy" diff --git a/modules/webhook/variables.tf b/modules/webhook/variables.tf index 2e5fafd205..d97e9db88b 100644 --- a/modules/webhook/variables.tf +++ b/modules/webhook/variables.tf @@ -234,6 +234,49 @@ variable "matcher_config_parameter_store_tier" { } } +variable "storage_provider" { + description = "Selected storage-provider type and opaque capabilities used by the webhook and optional dispatcher Lambdas." + type = object({ + type = optional(string, "aws_ssm") + direct = object({ + environment_variables = map(string) + iam_policy_json = optional(string, null) + }) + eventbridge = object({ + webhook = object({ + environment_variables = map(string) + iam_policy_json = optional(string, null) + }) + dispatcher = object({ + environment_variables = map(string) + iam_policy_json = optional(string, null) + }) + }) + }) + default = { + type = "aws_ssm" + direct = { + environment_variables = {} + iam_policy_json = null + } + eventbridge = { + webhook = { + environment_variables = {} + iam_policy_json = null + } + dispatcher = { + environment_variables = {} + iam_policy_json = null + } + } + } + + validation { + condition = contains(["aws_ssm", "aws_dynamodb"], var.storage_provider.type) + error_message = "storage_provider.type must be aws_ssm or aws_dynamodb." + } +} + variable "eventbridge" { description = <