diff --git a/README.md b/README.md index d028682a66..1c7dec9270 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,7 @@ Join our discord community via [this invite link](https://discord.gg/bxgXW8jJGh) | [instance\_max\_spot\_price](#input\_instance\_max\_spot\_price) | Max price price for spot instances per hour. This variable will be passed to the create fleet as max spot price for the fleet. | `string` | `null` | no | | [instance\_profile\_path](#input\_instance\_profile\_path) | The path that will be added to the instance\_profile, if not set the environment name will be used. | `string` | `null` | no | | [instance\_target\_capacity\_type](#input\_instance\_target\_capacity\_type) | Default lifecycle used for runner instances, can be either `spot` or `on-demand`. | `string` | `"spot"` | no | -| [instance\_termination\_watcher](#input\_instance\_termination\_watcher) | Configuration for the instance termination watcher. This feature is Beta, changes will not trigger a major release as long in beta.

`enable`: Enable or disable the spot termination watcher.
'features': Enable or disable features of the termination watcher.
`memory_size`: Memory size limit in MB of the lambda.
`s3_key`: S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas.
`s3_object_version`: S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket.
`timeout`: Time out of the lambda in seconds.
`zip`: File location of the lambda zip file. | 
object({
    enable = optional(bool, false)
    features = optional(object({
      enable_spot_termination_handler              = optional(bool, true)
      enable_spot_termination_notification_watcher = optional(bool, true)
    }), {})
    memory_size       = optional(number, null)
    s3_key            = optional(string, null)
    s3_object_version = optional(string, null)
    timeout           = optional(number, null)
    zip               = optional(string, null)
  })
| `{}` | no | +| [instance\_termination\_watcher](#input\_instance\_termination\_watcher) | Configuration for the instance termination watcher. This feature is Beta, changes will not trigger a major release as long in beta.

`enable`: Enable or disable the spot termination watcher.
'features': Enable or disable features of the termination watcher.
`enable_runner_deregistration`: Enable or disable deregistering the runner from GitHub when its EC2 instance is terminated.
`memory_size`: Memory size limit in MB of the lambda.
`s3_key`: S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas.
`s3_object_version`: S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket.
`timeout`: Time out of the lambda in seconds.
`zip`: File location of the lambda zip file. | 
object({
    enable = optional(bool, false)
    features = optional(object({
      enable_spot_termination_handler              = optional(bool, true)
      enable_spot_termination_notification_watcher = optional(bool, true)
    }), {})
    enable_runner_deregistration = optional(bool, true)
    memory_size                  = optional(number, null)
    s3_key                       = optional(string, null)
    s3_object_version            = optional(string, null)
    timeout                      = optional(number, null)
    zip                          = optional(string, null)
  })
| `{}` | no | | [instance\_types](#input\_instance\_types) | List of instance types for the action runner. Defaults are based on runner\_os (al2023 for linux, macOS Sequoia for osx, Windows Server Core for win). | `list(string)` | 
[
  "m5.large",
  "c5.large"
]
| no | | [job\_queue\_retention\_in\_seconds](#input\_job\_queue\_retention\_in\_seconds) | The number of seconds the job is held in the queue before it is purged. | `number` | `86400` | no | | [job\_retry](#input\_job\_retry) | Experimental! Can be removed / changed without trigger a major release.Configure job retries. The configuration enables job retries (for ephemeral runners). After creating the instances a message will be published to a job retry queue. The job retry check lambda is checking after a delay if the job is queued. If not the message will be published again on the scale-up (build queue). Using this feature can impact the rate limit of the GitHub app.

`enable`: Enable or disable the job retry feature.
`delay_in_seconds`: The delay in seconds before the job retry check lambda will check the job status.
`delay_backoff`: The backoff factor for the delay.
`lambda_memory_size`: Memory size limit in MB for the job retry check lambda.
`lambda_timeout`: Time out of the job retry check lambda in seconds.
`max_attempts`: The maximum number of attempts to retry the job. | 
object({
    enable             = optional(bool, false)
    delay_in_seconds   = optional(number, 300)
    delay_backoff      = optional(number, 2)
    lambda_memory_size = optional(number, 256)
    lambda_timeout     = optional(number, 30)
    max_attempts       = optional(number, 1)
  })
| `{}` | no | diff --git a/lambdas/functions/termination-watcher/package.json b/lambdas/functions/termination-watcher/package.json index e557d057cd..87622843a9 100644 --- a/lambdas/functions/termination-watcher/package.json +++ b/lambdas/functions/termination-watcher/package.json @@ -24,8 +24,15 @@ }, "dependencies": { "@aws-github-runner/aws-powertools-util": "*", + "@aws-github-runner/aws-ssm-util": "*", "@aws-sdk/client-ec2": "^3.1009.0", - "@middy/core": "^6.4.5" + "@aws-sdk/client-sqs": "^3.1009.0", + "@middy/core": "^6.4.5", + "@octokit/auth-app": "8.2.0", + "@octokit/core": "7.0.6", + "@octokit/plugin-throttling": "11.0.3", + "@octokit/request": "^9.2.2", + "@octokit/rest": "22.0.1" }, "nx": { "includedScripts": [ diff --git a/lambdas/functions/termination-watcher/src/ConfigResolver.test.ts b/lambdas/functions/termination-watcher/src/ConfigResolver.test.ts index 9aebb0588f..a614399066 100644 --- a/lambdas/functions/termination-watcher/src/ConfigResolver.test.ts +++ b/lambdas/functions/termination-watcher/src/ConfigResolver.test.ts @@ -37,6 +37,8 @@ describe('Test ConfigResolver', () => { delete process.env.ENABLE_METRICS_SPOT_WARNING; delete process.env.PREFIX; delete process.env.TAG_FILTERS; + delete process.env.ENABLE_RUNNER_DEREGISTRATION; + delete process.env.GHES_URL; }); it(description, async () => { @@ -55,4 +57,29 @@ describe('Test ConfigResolver', () => { expect(config.tagFilters).toEqual(output.tagFilters); }); }); + + describe('runner deregistration config', () => { + beforeEach(() => { + delete process.env.ENABLE_RUNNER_DEREGISTRATION; + delete process.env.GHES_URL; + }); + + it('should default to disabled', () => { + const config = new Config(); + expect(config.enableRunnerDeregistration).toBe(false); + expect(config.ghesApiUrl).toBe(''); + }); + + it('should enable deregistration when env var is true', () => { + process.env.ENABLE_RUNNER_DEREGISTRATION = 'true'; + const config = new Config(); + expect(config.enableRunnerDeregistration).toBe(true); + }); + + it('should set GHES URL when provided', () => { + process.env.GHES_URL = 'https://github.internal.co/api/v3'; + const config = new Config(); + expect(config.ghesApiUrl).toBe('https://github.internal.co/api/v3'); + }); + }); }); diff --git a/lambdas/functions/termination-watcher/src/ConfigResolver.ts b/lambdas/functions/termination-watcher/src/ConfigResolver.ts index 9e98b2a20a..949cefc9fb 100644 --- a/lambdas/functions/termination-watcher/src/ConfigResolver.ts +++ b/lambdas/functions/termination-watcher/src/ConfigResolver.ts @@ -5,6 +5,8 @@ export class Config { createSpotTerminationMetric: boolean; tagFilters: Record; prefix: string; + enableRunnerDeregistration: boolean; + ghesApiUrl: string; constructor() { const logger = createChildLogger('config-resolver'); @@ -14,6 +16,8 @@ export class Config { this.createSpotWarningMetric = process.env.ENABLE_METRICS_SPOT_WARNING === 'true'; this.createSpotTerminationMetric = process.env.ENABLE_METRICS_SPOT_TERMINATION === 'true'; this.prefix = process.env.PREFIX ?? ''; + this.enableRunnerDeregistration = process.env.ENABLE_RUNNER_DEREGISTRATION === 'true'; + this.ghesApiUrl = process.env.GHES_URL ?? ''; this.tagFilters = { 'ghr:environment': this.prefix }; const rawTagFilters = process.env.TAG_FILTERS; diff --git a/lambdas/functions/termination-watcher/src/deregister.test.ts b/lambdas/functions/termination-watcher/src/deregister.test.ts new file mode 100644 index 0000000000..dfc2854252 --- /dev/null +++ b/lambdas/functions/termination-watcher/src/deregister.test.ts @@ -0,0 +1,295 @@ +import { Instance } from '@aws-sdk/client-ec2'; +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { deregisterRunner, createThrottleOptions } from './deregister'; +import { Config } from './ConfigResolver'; +import type { EndpointDefaults } from '@octokit/types'; + +const mockGetParameter = vi.fn(); +vi.mock('@aws-github-runner/aws-ssm-util', () => ({ + getParameter: (...args: unknown[]) => mockGetParameter(...args), +})); + +const mockCreateAppAuth = vi.fn(); +vi.mock('@octokit/auth-app', () => ({ + createAppAuth: (...args: unknown[]) => mockCreateAppAuth(...args), +})); + +const mockPaginate = { + iterator: vi.fn(), +}; + +const mockActions = { + listSelfHostedRunnersForOrg: vi.fn(), + listSelfHostedRunnersForRepo: vi.fn(), + deleteSelfHostedRunnerFromOrg: vi.fn(), + deleteSelfHostedRunnerFromRepo: vi.fn(), +}; + +const mockApps = { + getOrgInstallation: vi.fn(), + getRepoInstallation: vi.fn(), +}; + +function MockOctokit() { + return { + actions: mockActions, + apps: mockApps, + paginate: mockPaginate, + }; +} +MockOctokit.plugin = vi.fn().mockReturnValue(MockOctokit); + +vi.mock('@octokit/rest', () => ({ + Octokit: MockOctokit, +})); + +vi.mock('@octokit/plugin-throttling', () => ({ + throttling: vi.fn(), +})); + +vi.mock('@octokit/request', () => ({ + request: { + defaults: vi.fn().mockReturnValue(vi.fn()), + }, +})); + +const baseConfig: Config = { + createSpotWarningMetric: false, + createSpotTerminationMetric: true, + tagFilters: { 'ghr:environment': 'test' }, + prefix: 'runners', + enableRunnerDeregistration: true, + ghesApiUrl: '', +}; + +const orgInstance: Instance = { + InstanceId: 'i-12345678901234567', + InstanceType: 't2.micro', + Tags: [ + { Key: 'Name', Value: 'test-instance' }, + { Key: 'ghr:environment', Value: 'test' }, + { Key: 'ghr:Owner', Value: 'test-org' }, + { Key: 'ghr:Type', Value: 'Org' }, + ], + State: { Name: 'running' }, + LaunchTime: new Date('2021-01-01'), +}; + +const repoInstance: Instance = { + InstanceId: 'i-repo12345678901234', + InstanceType: 't2.micro', + Tags: [ + { Key: 'Name', Value: 'test-repo-instance' }, + { Key: 'ghr:environment', Value: 'test' }, + { Key: 'ghr:Owner', Value: 'test-org/test-repo' }, + { Key: 'ghr:Type', Value: 'Repo' }, + ], + State: { Name: 'running' }, + LaunchTime: new Date('2021-01-01'), +}; + +function setupAuthMocks() { + const appPrivateKey = Buffer.from('fake-private-key').toString('base64'); + mockGetParameter.mockImplementation((name: string) => { + if (name === 'github-app-id') return Promise.resolve('12345'); + if (name === 'github-app-key') return Promise.resolve(appPrivateKey); + return Promise.reject(new Error(`Unknown parameter: ${name}`)); + }); + + // App auth returns app token + const mockAuth = vi.fn(); + mockAuth.mockImplementation((opts: { type: string }) => { + if (opts.type === 'app') { + return Promise.resolve({ token: 'app-token' }); + } + return Promise.resolve({ token: 'installation-token' }); + }); + mockCreateAppAuth.mockReturnValue(mockAuth); +} + +describe('deregisterRunner', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env.PARAMETER_GITHUB_APP_ID_NAME = 'github-app-id'; + process.env.PARAMETER_GITHUB_APP_KEY_BASE64_NAME = 'github-app-key'; + setupAuthMocks(); + }); + + it('should skip deregistration when disabled', async () => { + await deregisterRunner(orgInstance, { ...baseConfig, enableRunnerDeregistration: false }); + expect(mockGetParameter).not.toHaveBeenCalled(); + }); + + it('should skip deregistration when instance ID is missing', async () => { + const instance: Instance = { ...orgInstance, InstanceId: undefined }; + await deregisterRunner(instance, baseConfig); + expect(mockGetParameter).not.toHaveBeenCalled(); + }); + + it('should skip deregistration when ghr:Owner tag is missing', async () => { + const instance: Instance = { + ...orgInstance, + Tags: [{ Key: 'Name', Value: 'test' }], + }; + await deregisterRunner(instance, baseConfig); + // Auth should not be called since we bail early + expect(mockCreateAppAuth).not.toHaveBeenCalled(); + }); + + it('should deregister an org runner successfully', async () => { + mockApps.getOrgInstallation.mockResolvedValue({ data: { id: 999 } }); + + async function* fakeIterator() { + yield { data: [{ id: 42, name: `runner-i-12345678901234567` }] }; + } + mockPaginate.iterator.mockReturnValue(fakeIterator()); + + mockActions.deleteSelfHostedRunnerFromOrg.mockResolvedValue({}); + + await deregisterRunner(orgInstance, baseConfig); + + expect(mockApps.getOrgInstallation).toHaveBeenCalledWith({ org: 'test-org' }); + expect(mockActions.deleteSelfHostedRunnerFromOrg).toHaveBeenCalledWith({ + org: 'test-org', + runner_id: 42, + }); + }); + + it('should deregister a repo runner successfully', async () => { + mockApps.getRepoInstallation.mockResolvedValue({ data: { id: 888 } }); + + async function* fakeIterator() { + yield { data: [{ id: 55, name: `runner-i-repo12345678901234` }] }; + } + mockPaginate.iterator.mockReturnValue(fakeIterator()); + + mockActions.deleteSelfHostedRunnerFromRepo.mockResolvedValue({}); + + await deregisterRunner(repoInstance, baseConfig); + + expect(mockApps.getRepoInstallation).toHaveBeenCalledWith({ owner: 'test-org', repo: 'test-repo' }); + expect(mockActions.deleteSelfHostedRunnerFromRepo).toHaveBeenCalledWith({ + owner: 'test-org', + repo: 'test-repo', + runner_id: 55, + }); + }); + + it('should handle runner not found gracefully', async () => { + mockApps.getOrgInstallation.mockResolvedValue({ data: { id: 999 } }); + + async function* fakeIterator() { + yield { data: [{ id: 42, name: 'runner-other-instance' }] }; + } + mockPaginate.iterator.mockReturnValue(fakeIterator()); + + await deregisterRunner(orgInstance, baseConfig); + + expect(mockActions.deleteSelfHostedRunnerFromOrg).not.toHaveBeenCalled(); + }); + + it('should handle GitHub API errors gracefully', async () => { + mockApps.getOrgInstallation.mockRejectedValue(new Error('GitHub API error')); + + await deregisterRunner(orgInstance, baseConfig); + + // Should not throw — error is caught internally + expect(mockActions.deleteSelfHostedRunnerFromOrg).not.toHaveBeenCalled(); + }); + + it('should default to Org runner type when ghr:Type tag is missing', async () => { + const instance: Instance = { + ...orgInstance, + Tags: [ + { Key: 'ghr:environment', Value: 'test' }, + { Key: 'ghr:Owner', Value: 'test-org' }, + ], + }; + + mockApps.getOrgInstallation.mockResolvedValue({ data: { id: 999 } }); + + async function* fakeIterator() { + yield { data: [{ id: 42, name: `runner-i-12345678901234567` }] }; + } + mockPaginate.iterator.mockReturnValue(fakeIterator()); + + mockActions.deleteSelfHostedRunnerFromOrg.mockResolvedValue({}); + + await deregisterRunner(instance, baseConfig); + + expect(mockApps.getOrgInstallation).toHaveBeenCalledWith({ org: 'test-org' }); + expect(mockActions.deleteSelfHostedRunnerFromOrg).toHaveBeenCalledWith({ + org: 'test-org', + runner_id: 42, + }); + }); + + it('should use GHES API URL when configured', async () => { + const ghesConfig = { ...baseConfig, ghesApiUrl: 'https://github.internal.co/api/v3' }; + + mockApps.getOrgInstallation.mockResolvedValue({ data: { id: 999 } }); + + async function* fakeIterator() { + yield { data: [{ id: 42, name: `runner-i-12345678901234567` }] }; + } + mockPaginate.iterator.mockReturnValue(fakeIterator()); + + mockActions.deleteSelfHostedRunnerFromOrg.mockResolvedValue({}); + + await deregisterRunner(orgInstance, ghesConfig); + + expect(mockActions.deleteSelfHostedRunnerFromOrg).toHaveBeenCalled(); + }); + + it('should paginate through multiple pages to find runner', async () => { + mockApps.getOrgInstallation.mockResolvedValue({ data: { id: 999 } }); + + async function* fakeIterator() { + yield { data: [{ id: 1, name: 'runner-other-1' }] }; + yield { data: [{ id: 2, name: 'runner-other-2' }] }; + yield { data: [{ id: 42, name: `runner-i-12345678901234567` }] }; + } + mockPaginate.iterator.mockReturnValue(fakeIterator()); + + mockActions.deleteSelfHostedRunnerFromOrg.mockResolvedValue({}); + + await deregisterRunner(orgInstance, baseConfig); + + expect(mockActions.deleteSelfHostedRunnerFromOrg).toHaveBeenCalledWith({ + org: 'test-org', + runner_id: 42, + }); + }); + + it('should handle repo runner not found gracefully', async () => { + mockApps.getRepoInstallation.mockResolvedValue({ data: { id: 888 } }); + + async function* fakeIterator() { + yield { data: [{ id: 99, name: 'runner-other-instance' }] }; + } + mockPaginate.iterator.mockReturnValue(fakeIterator()); + + await deregisterRunner(repoInstance, baseConfig); + + expect(mockActions.deleteSelfHostedRunnerFromRepo).not.toHaveBeenCalled(); + }); + + it('should handle instance with no tags', async () => { + const instance: Instance = { + InstanceId: 'i-12345678901234567', + Tags: undefined, + }; + await deregisterRunner(instance, baseConfig); + expect(mockCreateAppAuth).not.toHaveBeenCalled(); + }); +}); + +describe('createThrottleOptions', () => { + it('should return false for rate limit and log warning', () => { + const options = createThrottleOptions(); + const endpointDefaults = { method: 'GET', url: '/test' } as Required; + + expect(options.onRateLimit(60, endpointDefaults)).toBe(false); + expect(options.onSecondaryRateLimit(60, endpointDefaults)).toBe(false); + }); +}); diff --git a/lambdas/functions/termination-watcher/src/deregister.ts b/lambdas/functions/termination-watcher/src/deregister.ts new file mode 100644 index 0000000000..ea53ad5240 --- /dev/null +++ b/lambdas/functions/termination-watcher/src/deregister.ts @@ -0,0 +1,287 @@ +import { createAppAuth } from '@octokit/auth-app'; +import { Octokit } from '@octokit/rest'; +import { throttling } from '@octokit/plugin-throttling'; +import { request } from '@octokit/request'; +import { Instance } from '@aws-sdk/client-ec2'; +import { SQSClient, SendMessageCommand } from '@aws-sdk/client-sqs'; +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; +import { getParameter } from '@aws-github-runner/aws-ssm-util'; +import type { EndpointDefaults } from '@octokit/types'; +import type { Config } from './ConfigResolver'; + +export interface DeregisterRetryMessage { + instanceId: string; + owner: string; + runnerType: string; + runnerId: number; + retryCount: number; +} + +const sqsClient = new SQSClient({ region: process.env.AWS_REGION }); + +const logger = createChildLogger('deregister'); + +export function createThrottleOptions() { + return { + onRateLimit: (_retryAfter: number, options: Required) => { + logger.warn(`Rate limit hit for ${options.method} ${options.url}`); + return false; + }, + onSecondaryRateLimit: (_retryAfter: number, options: Required) => { + logger.warn(`Secondary rate limit hit for ${options.method} ${options.url}`); + return false; + }, + }; +} + +async function getAppCredentials(): Promise<{ appId: number; privateKey: string }> { + const appId = parseInt(await getParameter(process.env.PARAMETER_GITHUB_APP_ID_NAME!)); + const privateKey = Buffer.from(await getParameter(process.env.PARAMETER_GITHUB_APP_KEY_BASE64_NAME!), 'base64') + .toString() + .replace('/[\\n]/g', String.fromCharCode(10)); + return { appId, privateKey }; +} + +function createOctokitInstance(token: string, ghesApiUrl: string): Octokit { + const CustomOctokit = Octokit.plugin(throttling); + const octokitOptions: ConstructorParameters[0] = { + auth: token, + }; + if (ghesApiUrl) { + octokitOptions.baseUrl = ghesApiUrl; + } + return new CustomOctokit({ + ...octokitOptions, + userAgent: 'github-aws-runners-termination-watcher', + throttle: createThrottleOptions(), + }); +} + +async function createAuthenticatedClient(ghesApiUrl: string): Promise { + const { appId, privateKey } = await getAppCredentials(); + const authOptions: { appId: number; privateKey: string; request?: typeof request } = { + appId, + privateKey, + }; + if (ghesApiUrl) { + authOptions.request = request.defaults({ baseUrl: ghesApiUrl }); + } + const auth = createAppAuth(authOptions); + const appAuth = await auth({ type: 'app' }); + return createOctokitInstance(appAuth.token, ghesApiUrl); +} + +function getOwnerFromTags(instance: Instance): string | undefined { + return instance.Tags?.find((tag) => tag.Key === 'ghr:Owner')?.Value; +} + +function getRunnerTypeFromTags(instance: Instance): string | undefined { + return instance.Tags?.find((tag) => tag.Key === 'ghr:Type')?.Value; +} + +async function getInstallationId(octokit: Octokit, owner: string): Promise { + const { data: installation } = await octokit.apps.getOrgInstallation({ org: owner }); + return installation.id; +} + +async function getInstallationIdForRepo(octokit: Octokit, owner: string, repo: string): Promise { + const { data: installation } = await octokit.apps.getRepoInstallation({ owner, repo }); + return installation.id; +} + +async function createInstallationClient( + appOctokit: Octokit, + owner: string, + runnerType: string, + ghesApiUrl: string, +): Promise { + let installationId: number; + if (runnerType === 'Repo') { + const [repoOwner, repo] = owner.split('/'); + installationId = await getInstallationIdForRepo(appOctokit, repoOwner, repo); + } else { + installationId = await getInstallationId(appOctokit, owner); + } + + const { appId, privateKey } = await getAppCredentials(); + const authOptions: { appId: number; privateKey: string; installationId: number; request?: typeof request } = { + appId, + privateKey, + installationId, + }; + if (ghesApiUrl) { + authOptions.request = request.defaults({ baseUrl: ghesApiUrl }); + } + const auth = createAppAuth(authOptions); + const installationAuth = await auth({ type: 'installation' }); + return createOctokitInstance(installationAuth.token, ghesApiUrl); +} + +async function findRunnerByInstanceId( + octokit: Octokit, + owner: string, + instanceId: string, + runnerType: string, +): Promise<{ id: number; name: string } | undefined> { + if (runnerType === 'Repo') { + const [repoOwner, repo] = owner.split('/'); + for await (const response of octokit.paginate.iterator(octokit.actions.listSelfHostedRunnersForRepo, { + owner: repoOwner, + repo, + per_page: 100, + })) { + const runner = response.data.find((r) => r.name.includes(instanceId)); + if (runner) { + return { id: runner.id, name: runner.name }; + } + } + } else { + for await (const response of octokit.paginate.iterator(octokit.actions.listSelfHostedRunnersForOrg, { + org: owner, + per_page: 100, + })) { + const runner = response.data.find((r) => r.name.includes(instanceId)); + if (runner) { + return { id: runner.id, name: runner.name }; + } + } + } + + return undefined; +} + +async function deleteRunner(octokit: Octokit, owner: string, runnerId: number, runnerType: string): Promise { + if (runnerType === 'Repo') { + const [repoOwner, repo] = owner.split('/'); + await octokit.actions.deleteSelfHostedRunnerFromRepo({ + owner: repoOwner, + repo, + runner_id: runnerId, + }); + } else { + await octokit.actions.deleteSelfHostedRunnerFromOrg({ + org: owner, + runner_id: runnerId, + }); + } +} + +export async function deregisterRunner(instance: Instance, config: Config): Promise { + if (!config.enableRunnerDeregistration) { + logger.debug('Runner deregistration is disabled, skipping'); + return; + } + + const instanceId = instance.InstanceId; + if (!instanceId) { + logger.warn('Instance ID is missing, cannot deregister runner'); + return; + } + + const owner = getOwnerFromTags(instance); + const runnerType = getRunnerTypeFromTags(instance) ?? 'Org'; + + if (!owner) { + logger.warn('ghr:Owner tag not found on instance, cannot deregister runner', { instanceId }); + return; + } + + try { + logger.info('Attempting to deregister runner from GitHub', { instanceId, owner, runnerType }); + + const appOctokit = await createAuthenticatedClient(config.ghesApiUrl); + const installationOctokit = await createInstallationClient(appOctokit, owner, runnerType, config.ghesApiUrl); + + const runner = await findRunnerByInstanceId(installationOctokit, owner, instanceId, runnerType); + if (!runner) { + logger.info('Runner not found in GitHub, may have already been deregistered', { instanceId, owner }); + return; + } + + await deleteRunner(installationOctokit, owner, runner.id, runnerType); + logger.info('Successfully deregistered runner from GitHub', { + instanceId, + runnerId: runner.id, + runnerName: runner.name, + owner, + }); + } catch (error) { + // GitHub returns 422 when a runner is currently executing a job. + // Queue a delayed retry — the instance will be terminated by EC2 shortly, + // and the runner will appear offline when we retry in 5 minutes. + const isRunnerBusy = error instanceof Error && 'status' in error && (error as { status: number }).status === 422; + if (isRunnerBusy) { + const queueUrl = process.env.DEREGISTER_RETRY_QUEUE_URL; + if (queueUrl) { + await queueDeregisterRetry(queueUrl, { instanceId, owner, runnerType, runnerId: 0, retryCount: 0 }); + logger.warn('Runner is busy — queued deregistration retry in 5 minutes via SQS', { instanceId, owner }); + } else { + logger.warn('Runner is busy and DEREGISTER_RETRY_QUEUE_URL is not set — deregistration skipped', { + instanceId, + owner, + }); + } + } else { + logger.error('Failed to deregister runner from GitHub', { + instanceId, + owner, + error: error as Error, + }); + } + } +} + +async function queueDeregisterRetry(queueUrl: string, message: DeregisterRetryMessage): Promise { + const command = new SendMessageCommand({ + QueueUrl: queueUrl, + MessageBody: JSON.stringify(message), + }); + await sqsClient.send(command); +} + +export async function handleDeregisterRetry(queueUrl: string, message: DeregisterRetryMessage): Promise { + const { instanceId, owner, runnerType, retryCount } = message; + logger.info('Processing deregistration retry from SQS', { instanceId, owner, runnerType, retryCount }); + + try { + const appOctokit = await createAuthenticatedClient(''); + const installationOctokit = await createInstallationClient(appOctokit, owner, runnerType, ''); + + const runner = await findRunnerByInstanceId(installationOctokit, owner, instanceId, runnerType); + if (!runner) { + logger.info('Runner not found in GitHub — already deregistered or never registered', { instanceId, owner }); + return; + } + + await deleteRunner(installationOctokit, owner, runner.id, runnerType); + logger.info('Successfully deregistered runner via SQS retry', { + instanceId, + runnerId: runner.id, + runnerName: runner.name, + owner, + retryCount, + }); + } catch (error) { + const isRunnerBusy = error instanceof Error && 'status' in error && (error as { status: number }).status === 422; + if (isRunnerBusy) { + // Re-enqueue for another retry — SQS maxReceiveCount DLQ will stop after 3 total attempts. + // Re-send explicitly so each retry resets the delay (SQS visibility timeout applies on re-receive, + // but re-sending gives us the full 5-minute DelaySeconds again). + await queueDeregisterRetry(queueUrl, { ...message, retryCount: retryCount + 1 }); + logger.warn('Runner still busy on retry — re-queued for another attempt', { + instanceId, + owner, + retryCount: retryCount + 1, + }); + } else { + logger.error('Failed to deregister runner on retry', { + instanceId, + owner, + retryCount, + error: error as Error, + }); + // Re-throw so SQS treats this as a failure and routes to DLQ after maxReceiveCount + throw error; + } + } +} diff --git a/lambdas/functions/termination-watcher/src/lambda.ts b/lambdas/functions/termination-watcher/src/lambda.ts index 77949dd954..eda8e8d688 100644 --- a/lambdas/functions/termination-watcher/src/lambda.ts +++ b/lambdas/functions/termination-watcher/src/lambda.ts @@ -1,10 +1,11 @@ import middy from '@middy/core'; import { captureLambdaHandler, logger, metrics, setContext, tracer } from '@aws-github-runner/aws-powertools-util'; import { logMetrics } from '@aws-lambda-powertools/metrics/middleware'; -import { Context } from 'aws-lambda'; +import { Context, SQSEvent } from 'aws-lambda'; import { handle as handleTerminationWarning } from './termination-warning'; import { handle as handleTermination } from './termination'; +import { handleDeregisterRetry, DeregisterRetryMessage } from './deregister'; import { BidEvictedDetail, BidEvictedEvent, SpotInterruptionWarning, SpotTerminationDetail } from './types'; import { Config } from './ConfigResolver'; @@ -37,6 +38,29 @@ export async function termination(event: BidEvictedEvent, cont } } +export async function deregisterRetry(event: SQSEvent, context: Context): Promise { + setContext(context, 'lambda.ts'); + logger.logEventIfEnabled(event); + logger.debug('Processing SQS deregister retry batch', { recordCount: event.Records.length }); + + const queueUrl = process.env.DEREGISTER_RETRY_QUEUE_URL; + if (!queueUrl) { + logger.error('DEREGISTER_RETRY_QUEUE_URL is not set — cannot process retry messages'); + return; + } + + for (const record of event.Records) { + try { + const message = JSON.parse(record.body) as DeregisterRetryMessage; + await handleDeregisterRetry(queueUrl, message); + } catch (e) { + logger.error(`Failed to process SQS record ${record.messageId}`, { error: e as Error }); + // Re-throw to mark the message as failed so SQS can retry or route to DLQ + throw e; + } + } +} + const addMiddleware = () => { const middleware = middy(interruptionWarning); diff --git a/lambdas/functions/termination-watcher/src/metric-event.ts b/lambdas/functions/termination-watcher/src/metric-event.ts index ece33213a6..c3d8201972 100644 --- a/lambdas/functions/termination-watcher/src/metric-event.ts +++ b/lambdas/functions/termination-watcher/src/metric-event.ts @@ -13,7 +13,7 @@ export async function metricEvent( const instanceRunningTimeInSeconds = instance.LaunchTime ? (new Date(event.time).getTime() - new Date(instance.LaunchTime).getTime()) / 1000 : undefined; - logger.info(`Received spot notification for ${metricName}`, { + logger.info(`Received spot notification${metricName ? ` for ${metricName}` : ''}`, { instanceId: instance.InstanceId, instanceType: instance.InstanceType ?? 'unknown', instanceName: instance.Tags?.find((tag) => tag.Key === 'Name')?.Value, diff --git a/lambdas/functions/termination-watcher/src/modules.d.ts b/lambdas/functions/termination-watcher/src/modules.d.ts index dd0eb932b0..d7d5c74e02 100644 --- a/lambdas/functions/termination-watcher/src/modules.d.ts +++ b/lambdas/functions/termination-watcher/src/modules.d.ts @@ -4,5 +4,9 @@ declare namespace NodeJS { ENVIRONMENT: string; PREFIX?: string; TAG_FILTERS?: string; + ENABLE_RUNNER_DEREGISTRATION?: 'true' | 'false'; + GHES_URL?: string; + PARAMETER_GITHUB_APP_ID_NAME?: string; + PARAMETER_GITHUB_APP_KEY_BASE64_NAME?: string; } } diff --git a/lambdas/functions/termination-watcher/src/termination-warning.test.ts b/lambdas/functions/termination-watcher/src/termination-warning.test.ts index a92c590ed0..e9dc4a05af 100644 --- a/lambdas/functions/termination-watcher/src/termination-warning.test.ts +++ b/lambdas/functions/termination-watcher/src/termination-warning.test.ts @@ -4,6 +4,7 @@ import 'aws-sdk-client-mock-jest'; import { handle } from './termination-warning'; import { SpotInterruptionWarning, SpotTerminationDetail } from './types'; import { metricEvent } from './metric-event'; +import { deregisterRunner } from './deregister'; import { getInstances } from './ec2'; import { describe, it, expect, beforeEach, vi } from 'vitest'; @@ -12,6 +13,10 @@ vi.mock('./metric-event', () => ({ metricEvent: vi.fn(), })); +vi.mock('./deregister', () => ({ + deregisterRunner: vi.fn(), +})); + vi.mock('./ec2', async (importOriginal) => { const actual = await importOriginal(); return { @@ -27,6 +32,8 @@ const config = { createSpotTerminationMetric: false, tagFilters: { 'ghr:environment': 'test' }, prefix: 'runners', + enableRunnerDeregistration: true, + ghesApiUrl: '', }; const event: SpotInterruptionWarning = { @@ -67,13 +74,16 @@ describe('handle termination warning', () => { expect(metricEvent).toHaveBeenCalled(); expect(metricEvent).toHaveBeenCalledWith(instance, event, 'SpotInterruptionWarning', expect.anything()); + expect(deregisterRunner).toHaveBeenCalledWith(instance, config); }); it('should log details and not create a metric', async () => { vi.mocked(getInstances).mockResolvedValue([instance]); - await handle(event, { ...config, createSpotWarningMetric: false }); + const noMetricConfig = { ...config, createSpotWarningMetric: false }; + await handle(event, noMetricConfig); expect(metricEvent).toHaveBeenCalledWith(instance, event, undefined, expect.anything()); + expect(deregisterRunner).toHaveBeenCalledWith(instance, noMetricConfig); }); it('should not create a metric if filter not matched.', async () => { @@ -84,8 +94,11 @@ describe('handle termination warning', () => { createSpotTerminationMetric: false, tagFilters: { 'ghr:environment': '_NO_MATCH_' }, prefix: 'runners', + enableRunnerDeregistration: true, + ghesApiUrl: '', }); expect(metricEvent).not.toHaveBeenCalled(); + expect(deregisterRunner).not.toHaveBeenCalled(); }); }); diff --git a/lambdas/functions/termination-watcher/src/termination-warning.ts b/lambdas/functions/termination-watcher/src/termination-warning.ts index 1bc3a9e0c9..8e5330be25 100644 --- a/lambdas/functions/termination-watcher/src/termination-warning.ts +++ b/lambdas/functions/termination-watcher/src/termination-warning.ts @@ -4,6 +4,7 @@ import { EC2Client, Instance } from '@aws-sdk/client-ec2'; import { Config } from './ConfigResolver'; import { tagFilter, getInstances } from './ec2'; import { metricEvent } from './metric-event'; +import { deregisterRunner } from './deregister'; const logger = createChildLogger('termination-warning'); @@ -26,6 +27,7 @@ async function createMetricForInstances( if (matchFilter) { metricEvent(instance, event, config.createSpotWarningMetric ? 'SpotInterruptionWarning' : undefined, logger); + await deregisterRunner(instance, config); } else { logger.debug( `Received spot termination notification warning but ` + diff --git a/lambdas/functions/termination-watcher/src/termination.test.ts b/lambdas/functions/termination-watcher/src/termination.test.ts index c8791f6701..31b16ec0a7 100644 --- a/lambdas/functions/termination-watcher/src/termination.test.ts +++ b/lambdas/functions/termination-watcher/src/termination.test.ts @@ -4,6 +4,7 @@ import 'aws-sdk-client-mock-jest'; import { handle } from './termination'; import { BidEvictedDetail, BidEvictedEvent } from './types'; import { metricEvent } from './metric-event'; +import { deregisterRunner } from './deregister'; import { getInstances } from './ec2'; import { describe, it, expect, beforeEach, vi } from 'vitest'; @@ -12,6 +13,10 @@ vi.mock('./metric-event', () => ({ metricEvent: vi.fn(), })); +vi.mock('./deregister', () => ({ + deregisterRunner: vi.fn(), +})); + vi.mock('./ec2', async (importOriginal) => { const actual = await importOriginal(); return { @@ -27,6 +32,8 @@ const config = { createSpotTerminationMetric: true, tagFilters: { 'ghr:environment': 'test' }, prefix: 'runners', + enableRunnerDeregistration: true, + ghesApiUrl: '', }; const event: BidEvictedEvent = { @@ -88,13 +95,16 @@ describe('handle termination warning', () => { expect(metricEvent).toHaveBeenCalled(); expect(metricEvent).toHaveBeenCalledWith(instance, event, 'SpotTermination', expect.anything()); + expect(deregisterRunner).toHaveBeenCalledWith(instance, config); }); it('should log details and not create a metric', async () => { vi.mocked(getInstances).mockResolvedValue([instance]); - await handle(event, { ...config, createSpotTerminationMetric: false }); + const noMetricConfig = { ...config, createSpotTerminationMetric: false }; + await handle(event, noMetricConfig); expect(metricEvent).toHaveBeenCalledWith(instance, event, undefined, expect.anything()); + expect(deregisterRunner).toHaveBeenCalledWith(instance, noMetricConfig); }); it('should not create a metric if filter not matched.', async () => { @@ -105,8 +115,11 @@ describe('handle termination warning', () => { createSpotTerminationMetric: true, tagFilters: { 'ghr:environment': '_NO_MATCH_' }, prefix: 'runners', + enableRunnerDeregistration: true, + ghesApiUrl: '', }); expect(metricEvent).not.toHaveBeenCalled(); + expect(deregisterRunner).not.toHaveBeenCalled(); }); }); diff --git a/lambdas/functions/termination-watcher/src/termination.ts b/lambdas/functions/termination-watcher/src/termination.ts index 4efc625245..e64228a443 100644 --- a/lambdas/functions/termination-watcher/src/termination.ts +++ b/lambdas/functions/termination-watcher/src/termination.ts @@ -4,6 +4,7 @@ import { EC2Client } from '@aws-sdk/client-ec2'; import { Config } from './ConfigResolver'; import { metricEvent } from './metric-event'; import { getInstances, tagFilter } from './ec2'; +import { deregisterRunner } from './deregister'; const logger = createChildLogger('termination-handler'); @@ -30,6 +31,7 @@ async function createMetricForInstances( if (matchFilter) { metricEvent(instance, event, config.createSpotTerminationMetric ? 'SpotTermination' : undefined, logger); + await deregisterRunner(instance, config); } else { logger.debug( `Received spot termination but ` + diff --git a/lambdas/tsconfig.json b/lambdas/tsconfig.json index 8dee3cd66a..6d733d2ae0 100644 --- a/lambdas/tsconfig.json +++ b/lambdas/tsconfig.json @@ -14,7 +14,9 @@ "emitDecoratorMetadata": true, "forceConsistentCasingInFileNames": false, "resolveJsonModule": true, - "types": ["vitest/globals"] + "types": [ + "vitest/globals" + ], + "skipLibCheck": true } -} - +} \ No newline at end of file diff --git a/lambdas/vitest.base.config.ts b/lambdas/vitest.base.config.ts index f8082eda28..bdfd28d540 100644 --- a/lambdas/vitest.base.config.ts +++ b/lambdas/vitest.base.config.ts @@ -9,11 +9,11 @@ const defaultConfig = defineConfig({ include: ['**/src/**/*.ts'], exclude: ['**/*local*.ts', '**/*.d.ts', '**/*.test.ts', '**/node_modules/**'], all: true, - reportsDirectory: './coverage' + reportsDirectory: './coverage', }, globals: true, - watch: false - } + watch: false, + }, }); export default defaultConfig; diff --git a/lambdas/yarn.lock b/lambdas/yarn.lock index bcab276463..bfe69a07b9 100644 --- a/lambdas/yarn.lock +++ b/lambdas/yarn.lock @@ -197,9 +197,16 @@ __metadata: resolution: "@aws-github-runner/termination-watcher@workspace:functions/termination-watcher" dependencies: "@aws-github-runner/aws-powertools-util": "npm:*" + "@aws-github-runner/aws-ssm-util": "npm:*" "@aws-sdk/client-ec2": "npm:^3.1009.0" + "@aws-sdk/client-sqs": "npm:^3.1009.0" "@aws-sdk/types": "npm:^3.973.6" "@middy/core": "npm:^6.4.5" + "@octokit/auth-app": "npm:8.2.0" + "@octokit/core": "npm:7.0.6" + "@octokit/plugin-throttling": "npm:11.0.3" + "@octokit/request": "npm:^9.2.2" + "@octokit/rest": "npm:22.0.1" "@types/aws-lambda": "npm:^8.10.159" "@types/node": "npm:^22.19.3" "@vercel/ncc": "npm:^0.38.4" @@ -3557,6 +3564,16 @@ __metadata: languageName: node linkType: hard +"@octokit/endpoint@npm:^10.1.4": + version: 10.1.4 + resolution: "@octokit/endpoint@npm:10.1.4" + dependencies: + "@octokit/types": "npm:^14.0.0" + universal-user-agent: "npm:^7.0.2" + checksum: 10c0/bf7cca71a05dc4751df658588e32642e59c98768e7509521226b997ea4837e2d16efd35c391231c76d888226f4daf80e6a9f347dee01a69f490253654dada581 + languageName: node + linkType: hard + "@octokit/endpoint@npm:^11.0.2": version: 11.0.2 resolution: "@octokit/endpoint@npm:11.0.2" @@ -3667,6 +3684,15 @@ __metadata: languageName: node linkType: hard +"@octokit/request-error@npm:^6.1.8": + version: 6.1.8 + resolution: "@octokit/request-error@npm:6.1.8" + dependencies: + "@octokit/types": "npm:^14.0.0" + checksum: 10c0/02aa5bfebb5b1b9e152558b4a6f4f7dcb149b41538778ffe0fce3395fd0da5c0862311a78e94723435667581b2a58a7cefa458cf7aa19ae2948ae419276f7ee1 + languageName: node + linkType: hard + "@octokit/request-error@npm:^7.0.0, @octokit/request-error@npm:^7.0.2": version: 7.0.2 resolution: "@octokit/request-error@npm:7.0.2" @@ -3689,6 +3715,19 @@ __metadata: languageName: node linkType: hard +"@octokit/request@npm:^9.2.2": + version: 9.2.4 + resolution: "@octokit/request@npm:9.2.4" + dependencies: + "@octokit/endpoint": "npm:^10.1.4" + "@octokit/request-error": "npm:^6.1.8" + "@octokit/types": "npm:^14.0.0" + fast-content-type-parse: "npm:^2.0.0" + universal-user-agent: "npm:^7.0.2" + checksum: 10c0/783ddf004e89e9738a6b4196c38fc377f166196a9f39a4956c50d675310113cf7a8e1ed1ed3842ae1d222d990231d1361fc8cf96adea2740e7e4caad216f19ab + languageName: node + linkType: hard + "@octokit/rest@npm:22.0.1": version: 22.0.1 resolution: "@octokit/rest@npm:22.0.1" @@ -7219,6 +7258,13 @@ __metadata: languageName: node linkType: hard +"fast-content-type-parse@npm:^2.0.0": + version: 2.0.1 + resolution: "fast-content-type-parse@npm:2.0.1" + checksum: 10c0/e5ff87d75a35ae4cf377df1dca46ec49e7abbdc8513689676ecdef548b94900b50e66e516e64470035d79b9f7010ef15d98c24d8ae803a881363cc59e0715e19 + languageName: node + linkType: hard + "fast-content-type-parse@npm:^3.0.0": version: 3.0.0 resolution: "fast-content-type-parse@npm:3.0.0" diff --git a/main.tf b/main.tf index d4708cb331..236a71d057 100644 --- a/main.tf +++ b/main.tf @@ -370,24 +370,30 @@ module "ami_housekeeper" { locals { lambda_instance_termination_watcher = { - prefix = var.prefix - tags = local.tags - aws_partition = var.aws_partition - architecture = var.lambda_architecture - principals = var.lambda_principals - runtime = var.lambda_runtime - security_group_ids = var.lambda_security_group_ids - subnet_ids = var.lambda_subnet_ids - lambda_tags = var.lambda_tags - log_level = var.log_level - log_class = var.log_class - logging_kms_key_id = var.logging_kms_key_id - logging_retention_in_days = var.logging_retention_in_days - role_path = var.role_path - role_permissions_boundary = var.role_permissions_boundary - s3_bucket = var.lambda_s3_bucket - tracing_config = var.tracing_config - metrics = var.metrics + prefix = var.prefix + tags = local.tags + aws_partition = var.aws_partition + architecture = var.lambda_architecture + principals = var.lambda_principals + runtime = var.lambda_runtime + security_group_ids = var.lambda_security_group_ids + subnet_ids = var.lambda_subnet_ids + lambda_tags = var.lambda_tags + log_level = var.log_level + log_class = var.log_class + logging_kms_key_id = var.logging_kms_key_id + logging_retention_in_days = var.logging_retention_in_days + role_path = var.role_path + role_permissions_boundary = var.role_permissions_boundary + s3_bucket = var.lambda_s3_bucket + tracing_config = var.tracing_config + metrics = var.metrics + enable_runner_deregistration = var.instance_termination_watcher.enable_runner_deregistration + github_app_parameters = var.instance_termination_watcher.enable_runner_deregistration ? { + id = local.github_app_parameters.id + key_base64 = local.github_app_parameters.key_base64 + } : null + ghes_url = var.ghes_url } } diff --git a/modules/multi-runner/README.md b/modules/multi-runner/README.md index a5101e0a84..13085c2b3c 100644 --- a/modules/multi-runner/README.md +++ b/modules/multi-runner/README.md @@ -134,7 +134,7 @@ module "multi-runner" { | [github\_app](#input\_github\_app) | GitHub app parameters, see your github app.
You can optionally create the SSM parameters yourself and provide the ARN and name here, through the `*_ssm` attributes.
If you chose to provide the configuration values directly here,
please ensure the key is the base64-encoded `.pem` file (the output of `base64 app.private-key.pem`, not the content of `private-key.pem`).
Note: the provided SSM parameters arn and name have a precedence over the actual value (i.e `key_base64_ssm` has a precedence over `key_base64` etc). | 
object({
    key_base64 = optional(string)
    key_base64_ssm = optional(object({
      arn  = string
      name = string
    }))
    id = optional(string)
    id_ssm = optional(object({
      arn  = string
      name = string
    }))
    webhook_secret = optional(string)
    webhook_secret_ssm = optional(object({
      arn  = string
      name = string
    }))
  })
| n/a | yes | | [iam\_overrides](#input\_iam\_overrides) | This map provides the possibility to override some IAM defaults. The following attributes are supported: `instance_profile_name` overrides the instance profile name used in the launch template. `runner_role_arn` overrides the IAM role ARN used for the runner instances. | 
object({
    override_instance_profile = optional(bool, null)
    instance_profile_name     = optional(string, null)
    override_runner_role      = optional(bool, null)
    runner_role_arn           = optional(string, null)
  })
| 
{
  "instance_profile_name": null,
  "override_instance_profile": false,
  "override_runner_role": false,
  "runner_role_arn": null
}
| no | | [instance\_profile\_path](#input\_instance\_profile\_path) | The path that will be added to the instance\_profile, if not set the environment name will be used. | `string` | `null` | no | -| [instance\_termination\_watcher](#input\_instance\_termination\_watcher) | Configuration for the spot termination watcher lambda function. This feature is Beta, changes will not trigger a major release as long in beta.

`enable`: Enable or disable the spot termination watcher.
`memory_size`: Memory size limit in MB of the lambda.
`s3_key`: S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas.
`s3_object_version`: S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket.
`timeout`: Time out of the lambda in seconds.
`zip`: File location of the lambda zip file. | 
object({
    enable = optional(bool, false)
    features = optional(object({
      enable_spot_termination_handler              = optional(bool, true)
      enable_spot_termination_notification_watcher = optional(bool, true)
    }), {})
    memory_size       = optional(number, null)
    s3_key            = optional(string, null)
    s3_object_version = optional(string, null)
    timeout           = optional(number, null)
    zip               = optional(string, null)
  })
| `{}` | no | +| [instance\_termination\_watcher](#input\_instance\_termination\_watcher) | Configuration for the spot termination watcher lambda function. This feature is Beta, changes will not trigger a major release as long in beta.

`enable`: Enable or disable the spot termination watcher.
`enable_runner_deregistration`: Enable or disable deregistering the runner from GitHub when its EC2 instance is terminated.
`environment_variables`: Additional environment variables to merge into the Lambda configuration.
`memory_size`: Memory size limit in MB of the lambda.
`s3_key`: S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas.
`s3_object_version`: S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket.
`timeout`: Time out of the lambda in seconds.
`zip`: File location of the lambda zip file. | 
object({
    enable = optional(bool, false)
    features = optional(object({
      enable_spot_termination_handler              = optional(bool, true)
      enable_spot_termination_notification_watcher = optional(bool, true)
    }), {})
    enable_runner_deregistration = optional(bool, true)
    environment_variables        = optional(map(string), {})
    memory_size                  = optional(number, null)
    s3_key                       = optional(string, null)
    s3_object_version            = optional(string, null)
    timeout                      = optional(number, null)
    zip                          = optional(string, null)
  })
| `{}` | no | | [key\_name](#input\_key\_name) | Key pair name | `string` | `null` | no | | [kms\_key\_arn](#input\_kms\_key\_arn) | Optional CMK Key ARN to be used for Parameter Store. | `string` | `null` | no | | [lambda\_architecture](#input\_lambda\_architecture) | AWS Lambda architecture. Lambda functions using Graviton processors ('arm64') tend to have better price/performance than 'x86\_64' functions. | `string` | `"arm64"` | no | diff --git a/modules/multi-runner/termination-watcher.tf b/modules/multi-runner/termination-watcher.tf index 5ddd4495bb..31e51cd216 100644 --- a/modules/multi-runner/termination-watcher.tf +++ b/modules/multi-runner/termination-watcher.tf @@ -1,23 +1,30 @@ locals { lambda_instance_termination_watcher = { - prefix = var.prefix - tags = local.tags - aws_partition = var.aws_partition - architecture = var.lambda_architecture - principals = var.lambda_principals - runtime = var.lambda_runtime - security_group_ids = var.lambda_security_group_ids - subnet_ids = var.lambda_subnet_ids - log_level = var.log_level - log_class = var.log_class - logging_kms_key_id = var.logging_kms_key_id - logging_retention_in_days = var.logging_retention_in_days - role_path = var.role_path - role_permissions_boundary = var.role_permissions_boundary - s3_bucket = var.lambda_s3_bucket - tracing_config = var.tracing_config - lambda_tags = var.lambda_tags - metrics = var.metrics + prefix = var.prefix + tags = local.tags + aws_partition = var.aws_partition + architecture = var.lambda_architecture + principals = var.lambda_principals + runtime = var.lambda_runtime + security_group_ids = var.lambda_security_group_ids + subnet_ids = var.lambda_subnet_ids + log_level = var.log_level + log_class = var.log_class + logging_kms_key_id = var.logging_kms_key_id + logging_retention_in_days = var.logging_retention_in_days + role_path = var.role_path + role_permissions_boundary = var.role_permissions_boundary + s3_bucket = var.lambda_s3_bucket + tracing_config = var.tracing_config + lambda_tags = var.lambda_tags + metrics = var.metrics + enable_runner_deregistration = var.instance_termination_watcher.enable_runner_deregistration + github_app_parameters = var.instance_termination_watcher.enable_runner_deregistration ? { + id = local.github_app_parameters.id + key_base64 = local.github_app_parameters.key_base64 + } : null + ghes_url = var.ghes_url + environment_variables = var.instance_termination_watcher.environment_variables } } diff --git a/modules/multi-runner/variables.tf b/modules/multi-runner/variables.tf index ace6654d29..ec9af676e8 100644 --- a/modules/multi-runner/variables.tf +++ b/modules/multi-runner/variables.tf @@ -726,6 +726,8 @@ variable "instance_termination_watcher" { Configuration for the spot termination watcher lambda function. This feature is Beta, changes will not trigger a major release as long in beta. `enable`: Enable or disable the spot termination watcher. + `enable_runner_deregistration`: Enable or disable deregistering the runner from GitHub when its EC2 instance is terminated. + `environment_variables`: Additional environment variables to merge into the Lambda configuration. `memory_size`: Memory size limit in MB of the lambda. `s3_key`: S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas. `s3_object_version`: S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket. @@ -739,11 +741,13 @@ variable "instance_termination_watcher" { enable_spot_termination_handler = optional(bool, true) enable_spot_termination_notification_watcher = optional(bool, true) }), {}) - memory_size = optional(number, null) - s3_key = optional(string, null) - s3_object_version = optional(string, null) - timeout = optional(number, null) - zip = optional(string, null) + enable_runner_deregistration = optional(bool, true) + environment_variables = optional(map(string), {}) + memory_size = optional(number, null) + s3_key = optional(string, null) + s3_object_version = optional(string, null) + timeout = optional(number, null) + zip = optional(string, null) }) default = {} } diff --git a/modules/termination-watcher/README.md b/modules/termination-watcher/README.md index dc6049ffec..4cdf37f13b 100644 --- a/modules/termination-watcher/README.md +++ b/modules/termination-watcher/README.md @@ -65,29 +65,42 @@ yarn run dist ## Providers -No providers. +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 6.21 | ## Modules | Name | Source | Version | |------|--------|---------| +| [deregister\_retry\_lambda](#module\_deregister\_retry\_lambda) | ../lambda | n/a | | [termination\_handler](#module\_termination\_handler) | ./termination | n/a | | [termination\_notification](#module\_termination\_notification) | ./notification | n/a | ## Resources -No resources. +| Name | Type | +|------|------| +| [aws_iam_role_policy.deregister_retry_ec2](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.deregister_retry_sqs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.deregister_retry_ssm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.notification_sqs_send](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_iam_role_policy.termination_sqs_send](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [aws_lambda_event_source_mapping.deregister_retry](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_event_source_mapping) | resource | +| [aws_sqs_queue.deregister_retry](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource | +| [aws_sqs_queue.deregister_retry_dlq](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sqs_queue) | resource | ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [config](#input\_config) | Configuration for the spot termination watcher.

`aws_partition`: Partition for the base arn if not 'aws'
`architecture`: AWS Lambda architecture. Lambda functions using Graviton processors ('arm64') tend to have better price/performance than 'x86\_64' functions.
`environment_variables`: Environment variables for the lambda.
'features': Features to enable the different lambda functions to handle spot termination events.
`lambda_principals`: Add extra principals to the role created for execution of the lambda, e.g. for local testing.
`lambda_tags`: Map of tags that will be added to created resources. By default resources will be tagged with name and environment.
`log_level`: Logging level for lambda logging. Valid values are 'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'.
`log_class`: The log class of the CloudWatch log group. Valid values are `STANDARD` or `INFREQUENT_ACCESS`.
`logging_kms_key_id`: Specifies the kms key id to encrypt the logs with
`logging_retention_in_days`: Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653.
`memory_size`: Memory size limit in MB of the lambda.
`prefix`: The prefix used for naming resources.
`role_path`: The path that will be added to the role, if not set the environment name will be used.
`role_permissions_boundary`: Permissions boundary that will be added to the created role for the lambda.
`runtime`: AWS Lambda runtime.
`s3_bucket`: S3 bucket from which to specify lambda functions. This is an alternative to providing local files directly.
`s3_key`: S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas.
`s3_object_version`: S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket.
`security_group_ids`: List of security group IDs associated with the Lambda function.
`subnet_ids`: List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`.
`tag_filters`: Map of tags that will be used to filter the resources to be tracked. Only for which all tags are present and starting with the same value as the value in the map will be tracked.
`tags`: Map of tags that will be added to created resources. By default resources will be tagged with name and environment.
`timeout`: Time out of the lambda in seconds.
`tracing_config`: Configuration for lambda tracing.
`zip`: File location of the lambda zip file. | 
object({
    aws_partition         = optional(string, null)
    architecture          = optional(string, null)
    environment_variables = optional(map(string), {})
    features = optional(object({
      enable_spot_termination_handler              = optional(bool, true)
      enable_spot_termination_notification_watcher = optional(bool, true)
    }), {})
    lambda_tags               = optional(map(string), {})
    log_level                 = optional(string, null)
    log_class                 = optional(string, "STANDARD")
    logging_kms_key_id        = optional(string, null)
    logging_retention_in_days = optional(number, null)
    memory_size               = optional(number, null)
    metrics = optional(object({
      enable    = optional(bool, false)
      namespace = optional(string, "GitHub Runners")
      metric = optional(object({
        enable_spot_termination         = optional(bool, true)
        enable_spot_termination_warning = optional(bool, true)
      }), {})
    }), {})
    prefix = optional(string, null)
    principals = optional(list(object({
      type        = string
      identifiers = list(string)
    })), [])
    role_path                 = optional(string, null)
    role_permissions_boundary = optional(string, null)
    runtime                   = optional(string, null)
    s3_bucket                 = optional(string, null)
    s3_key                    = optional(string, null)
    s3_object_version         = optional(string, null)
    security_group_ids        = optional(list(string), [])
    subnet_ids                = optional(list(string), [])
    tag_filters               = optional(map(string), null)
    tags                      = optional(map(string), {})
    timeout                   = optional(number, null)
    tracing_config = optional(object({
      mode                  = optional(string, null)
      capture_http_requests = optional(bool, false)
      capture_error         = optional(bool, false)
    }), {})
    zip = optional(string, null)
  })
| n/a | yes | +| [config](#input\_config) | Configuration for the spot termination watcher.

`aws_partition`: Partition for the base arn if not 'aws'
`architecture`: AWS Lambda architecture. Lambda functions using Graviton processors ('arm64') tend to have better price/performance than 'x86\_64' functions.
`environment_variables`: Environment variables for the lambda.
'features': Features to enable the different lambda functions to handle spot termination events.
`lambda_principals`: Add extra principals to the role created for execution of the lambda, e.g. for local testing.
`lambda_tags`: Map of tags that will be added to created resources. By default resources will be tagged with name and environment.
`log_level`: Logging level for lambda logging. Valid values are 'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'.
`log_class`: The log class of the CloudWatch log group. Valid values are `STANDARD` or `INFREQUENT_ACCESS`.
`logging_kms_key_id`: Specifies the kms key id to encrypt the logs with
`logging_retention_in_days`: Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653.
`memory_size`: Memory size limit in MB of the lambda.
`prefix`: The prefix used for naming resources.
`role_path`: The path that will be added to the role, if not set the environment name will be used.
`role_permissions_boundary`: Permissions boundary that will be added to the created role for the lambda.
`runtime`: AWS Lambda runtime.
`s3_bucket`: S3 bucket from which to specify lambda functions. This is an alternative to providing local files directly.
`s3_key`: S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas.
`s3_object_version`: S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket.
`security_group_ids`: List of security group IDs associated with the Lambda function.
`subnet_ids`: List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`.
`tag_filters`: Map of tags that will be used to filter the resources to be tracked. Only for which all tags are present and starting with the same value as the value in the map will be tracked.
`tags`: Map of tags that will be added to created resources. By default resources will be tagged with name and environment.
`timeout`: Time out of the lambda in seconds.
`tracing_config`: Configuration for lambda tracing.
`zip`: File location of the lambda zip file.
`enable_runner_deregistration`: Enable or disable deregistering the runner from GitHub when its EC2 instance is terminated.
`github_app_parameters`: GitHub App SSM parameters (`id` and `key_base64`, each a map of `arn`/`name`) used to authenticate to GitHub when deregistering runners.
`ghes_url`: GitHub Enterprise Server URL used to target the GHES API when deregistering runners. Leave `null` for github.com. | 
object({
    aws_partition         = optional(string, null)
    architecture          = optional(string, null)
    environment_variables = optional(map(string), {})
    features = optional(object({
      enable_spot_termination_handler              = optional(bool, true)
      enable_spot_termination_notification_watcher = optional(bool, true)
    }), {})
    lambda_tags               = optional(map(string), {})
    log_level                 = optional(string, null)
    log_class                 = optional(string, "STANDARD")
    logging_kms_key_id        = optional(string, null)
    logging_retention_in_days = optional(number, null)
    memory_size               = optional(number, null)
    metrics = optional(object({
      enable    = optional(bool, false)
      namespace = optional(string, "GitHub Runners")
      metric = optional(object({
        enable_spot_termination         = optional(bool, true)
        enable_spot_termination_warning = optional(bool, true)
      }), {})
    }), {})
    prefix = optional(string, null)
    principals = optional(list(object({
      type        = string
      identifiers = list(string)
    })), [])
    role_path                 = optional(string, null)
    role_permissions_boundary = optional(string, null)
    runtime                   = optional(string, null)
    s3_bucket                 = optional(string, null)
    s3_key                    = optional(string, null)
    s3_object_version         = optional(string, null)
    security_group_ids        = optional(list(string), [])
    subnet_ids                = optional(list(string), [])
    tag_filters               = optional(map(string), null)
    tags                      = optional(map(string), {})
    timeout                   = optional(number, null)
    tracing_config = optional(object({
      mode                  = optional(string, null)
      capture_http_requests = optional(bool, false)
      capture_error         = optional(bool, false)
    }), {})
    zip                          = optional(string, null)
    enable_runner_deregistration = optional(bool, false)
    github_app_parameters = optional(object({
      id         = map(string)
      key_base64 = map(string)
    }), null)
    ghes_url = optional(string, null)
  })
| n/a | yes | ## Outputs | Name | Description | |------|-------------| +| [deregister\_retry](#output\_deregister\_retry) | n/a | | [spot\_termination\_handler](#output\_spot\_termination\_handler) | n/a | | [spot\_termination\_notification](#output\_spot\_termination\_notification) | n/a | diff --git a/modules/termination-watcher/deregister-retry.tf b/modules/termination-watcher/deregister-retry.tf new file mode 100644 index 0000000000..921d7abf5e --- /dev/null +++ b/modules/termination-watcher/deregister-retry.tf @@ -0,0 +1,155 @@ +# SQS-based deregistration retry for runners that return 422 (busy executing a job). +# When a runner can't be deregistered immediately, the termination-watcher Lambda +# sends a message to this queue with a 5-minute delay. By the time the message +# becomes visible, the EC2 instance has terminated and the runner appears offline, +# allowing clean GitHub API deletion. + +# Dead-letter queue — messages that fail after 3 attempts land here for investigation +resource "aws_sqs_queue" "deregister_retry_dlq" { + count = local.enable_runner_deregistration ? 1 : 0 + + name = "${var.config.prefix}-deregister-retry-dlq" + message_retention_seconds = 1209600 # 14 days + tags = var.config.tags +} + +# Main retry queue — 5-minute delivery delay gives EC2 time to terminate +resource "aws_sqs_queue" "deregister_retry" { + count = local.enable_runner_deregistration ? 1 : 0 + + name = "${var.config.prefix}-deregister-retry" + delay_seconds = 300 # 5 minutes + message_retention_seconds = 86400 # 24 hours + visibility_timeout_seconds = 60 # Lambda timeout + buffer + tags = var.config.tags + + redrive_policy = jsonencode({ + deadLetterTargetArn = aws_sqs_queue.deregister_retry_dlq[0].arn + maxReceiveCount = 3 + }) +} + +# Dedicated Lambda function for processing SQS retry messages. +# Uses the same code package as the termination-watcher but with +# handler index.deregisterRetry (SQS event handler). +module "deregister_retry_lambda" { + count = local.enable_runner_deregistration ? 1 : 0 + source = "../lambda" + + lambda = merge(local.config, { + name = "deregister-retry" + handler = "index.deregisterRetry" + environment_variables = merge( + local.deregistration_env_vars, + var.config.environment_variables, + { + DEREGISTER_RETRY_QUEUE_URL = aws_sqs_queue.deregister_retry[0].url + TAG_FILTERS = jsonencode(var.config.tag_filters) + } + ) + }) +} + +# SQS event source mapping — triggers the retry Lambda when messages arrive +resource "aws_lambda_event_source_mapping" "deregister_retry" { + count = local.enable_runner_deregistration ? 1 : 0 + + event_source_arn = aws_sqs_queue.deregister_retry[0].arn + function_name = module.deregister_retry_lambda[0].lambda.function.arn + batch_size = 1 # Process one retry at a time to avoid GitHub rate limits + enabled = true +} + +# IAM: Allow the retry Lambda to receive/delete from the retry queue +resource "aws_iam_role_policy" "deregister_retry_sqs" { + count = local.enable_runner_deregistration ? 1 : 0 + + name = "sqs-deregister-retry" + role = module.deregister_retry_lambda[0].lambda.role.name + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:SendMessage" + ] + Resource = [ + aws_sqs_queue.deregister_retry[0].arn, + aws_sqs_queue.deregister_retry_dlq[0].arn + ] + } + ] + }) +} + +# IAM: Allow the retry Lambda to read SSM parameters (GitHub App credentials) +resource "aws_iam_role_policy" "deregister_retry_ssm" { + count = local.enable_runner_deregistration ? 1 : 0 + + name = "ssm-deregister-retry" + role = module.deregister_retry_lambda[0].lambda.role.name + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = ["ssm:GetParameter"] + Resource = local.ssm_parameter_arns + } + ] + }) +} + +# IAM: Allow the retry Lambda to describe EC2 instances (for tag lookups) +resource "aws_iam_role_policy" "deregister_retry_ec2" { + count = local.enable_runner_deregistration ? 1 : 0 + + name = "ec2-deregister-retry" + role = module.deregister_retry_lambda[0].lambda.role.name + + policy = templatefile("${path.module}/policies/lambda.json", {}) +} + +# IAM: Allow the notification Lambda to send messages to the retry queue +resource "aws_iam_role_policy" "notification_sqs_send" { + count = local.enable_runner_deregistration && var.config.features.enable_spot_termination_notification_watcher ? 1 : 0 + + name = "sqs-deregister-retry-send" + role = module.termination_notification[0].lambda.role.name + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = ["sqs:SendMessage"] + Resource = aws_sqs_queue.deregister_retry[0].arn + } + ] + }) +} + +# IAM: Allow the termination handler Lambda to send messages to the retry queue +resource "aws_iam_role_policy" "termination_sqs_send" { + count = local.enable_runner_deregistration && var.config.features.enable_spot_termination_handler ? 1 : 0 + + name = "sqs-deregister-retry-send" + role = module.termination_handler[0].lambda.role.name + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = ["sqs:SendMessage"] + Resource = aws_sqs_queue.deregister_retry[0].arn + } + ] + }) +} diff --git a/modules/termination-watcher/main.tf b/modules/termination-watcher/main.tf index 1cf8ccb275..919ba3a3e5 100644 --- a/modules/termination-watcher/main.tf +++ b/modules/termination-watcher/main.tf @@ -2,16 +2,35 @@ locals { lambda_zip = var.config.zip == null ? "${path.module}/../../lambdas/functions/termination-watcher/termination-watcher.zip" : var.config.zip name = "spot-termination-watcher" + enable_runner_deregistration = var.config.enable_runner_deregistration && var.config.github_app_parameters != null + + deregistration_env_vars = local.enable_runner_deregistration ? merge({ + ENABLE_RUNNER_DEREGISTRATION = "true" + PARAMETER_GITHUB_APP_ID_NAME = var.config.github_app_parameters.id.name + PARAMETER_GITHUB_APP_KEY_BASE64_NAME = var.config.github_app_parameters.key_base64.name + GHES_URL = var.config.ghes_url != null ? var.config.ghes_url : "" + }, length(aws_sqs_queue.deregister_retry) > 0 ? { + DEREGISTER_RETRY_QUEUE_URL = aws_sqs_queue.deregister_retry[0].url + } : {}) : {} + + ssm_parameter_arns = local.enable_runner_deregistration ? [ + var.config.github_app_parameters.id.arn, + var.config.github_app_parameters.key_base64.arn, + ] : [] + environment_variables = { ENABLE_METRICS_SPOT_WARNING = var.config.metrics != null ? var.config.metrics.enable && var.config.metrics.metric.enable_spot_termination_warning : false TAG_FILTERS = jsonencode(var.config.tag_filters) } config = merge(var.config, { - name = local.name, - handler = "index.interruptionWarning", - zip = local.lambda_zip, - environment_variables = local.environment_variables - metrics_namespace = var.config.metrics.namespace + name = local.name, + handler = "index.interruptionWarning", + zip = local.lambda_zip, + environment_variables = local.environment_variables + metrics_namespace = var.config.metrics.namespace + _deregistration_env_vars = local.deregistration_env_vars + _ssm_parameter_arns = local.ssm_parameter_arns + _enable_runner_deregistration = local.enable_runner_deregistration }) } diff --git a/modules/termination-watcher/notification/main.tf b/modules/termination-watcher/notification/main.tf index 82b961bc3c..735c34126b 100644 --- a/modules/termination-watcher/notification/main.tf +++ b/modules/termination-watcher/notification/main.tf @@ -4,10 +4,10 @@ locals { config = merge(var.config, { name = local.name, handler = "index.interruptionWarning", - environment_variables = { + environment_variables = merge({ ENABLE_METRICS_SPOT_WARNING = var.config.metrics != null ? var.config.metrics.enable && var.config.metrics.metric.enable_spot_termination_warning : false TAG_FILTERS = jsonencode(var.config.tag_filters) - } + }, var.config._deregistration_env_vars, var.config.environment_variables) }) } @@ -42,9 +42,67 @@ resource "aws_lambda_permission" "main" { source_arn = aws_cloudwatch_event_rule.spot_instance_termination_warning.arn } +# EC2 Instance State-change Notification — catches ALL termination types +# (scale-down, manual, spot reclamation, ASG) not just spot-specific events. +# Uses "shutting-down" state to deregister runners while instance metadata is still available. +# Reuses the same Lambda as the spot interruption warning handler since both event +# types have detail['instance-id'] — the handler extracts it identically. +resource "aws_cloudwatch_event_rule" "ec2_instance_state_change" { + count = var.config._enable_runner_deregistration ? 1 : 0 + + name = "${var.config.prefix != null ? format("%s-", var.config.prefix) : ""}instance-termination" + description = "EC2 Instance Termination (all causes) — deregisters runners from GitHub" + tags = local.config.tags + + event_pattern = <