feat(github-app): use ssm manifest for extra apps

docs/rate-limits-and-tuning.md

@@ -50,6 +50,17 @@ Sustained: 10,000/hour ÷ 60 = **~166 runners/minute**.
 
 Without a token cache, each runner also costs a `POST /app/installations/{id}/access_tokens` (5 points) against the `core` endpoint. This doesn't directly reduce JIT throughput (different endpoint) but competes with `isJobQueued` for the `core` hourly budget.
 
+### Distributing load across multiple GitHub Apps
+
+Rate limits are per App installation and cannot be raised. To scale beyond one App's budget, configure extra Apps with `additional_github_apps`. The control-plane lambdas select one App per invocation, making the effective limit N × the per-App limit.
+
+> [!IMPORTANT]
+> Every additional App must be installed on the same organizations or repositories as the primary App. The module cannot verify this. A missing installation surfaces at runtime as installation lookup 404s on the fraction of invocations that select the misconfigured App, which is hard to trace back to the installation.
+
+Only the primary App needs a webhook configured in GitHub; additional Apps are used for API calls only. Set `installation_id` per additional App to skip one installation lookup per invocation.
+
+The lambdas receive additional App credentials through a manifest SSM parameter that lists the per-App credential parameter names, so the lambda environment size stays constant regardless of App count.
+
 ### GHES
 
 Rate limits are **disabled by default** on GitHub Enterprise Server and must be explicitly enabled by the site admin. When enabled, the same formula applies.

lambdas/functions/control-plane/src/github/auth.test.ts

@@ -2,7 +2,7 @@ import { createAppAuth } from '@octokit/auth-app';
 import { StrategyOptions } from '@octokit/auth-app/dist-types/types';
 import { request } from '@octokit/request';
 import { RequestInterface, RequestParameters } from '@octokit/types';
-import { getParameters } from '@aws-github-runner/aws-ssm-util';
+import { getParameter, getParameters } from '@aws-github-runner/aws-ssm-util';
 import { generateKeyPairSync } from 'node:crypto';
 import * as nock from 'nock';
 
@@ -35,6 +35,7 @@ const PARAMETER_GITHUB_APP_ID_NAME = `/actions-runner/${ENVIRONMENT}/github_app_
 const PARAMETER_GITHUB_APP_KEY_BASE64_NAME = `/actions-runner/${ENVIRONMENT}/github_app_key_base64`;
 
 const mockedGetParameters = vi.mocked(getParameters);
+const mockedGetParameter = vi.mocked(getParameter);
 
 beforeEach(() => {
   vi.resetModules();
@@ -341,23 +342,32 @@ describe('Test getStoredInstallationId', () => {
     vi.mocked(createAppAuth).mockReturnValue(mockWithHook);
   });
 
-  it('returns stored installation ID when configured', async () => {
-    const installationIdParam = `/actions-runner/${ENVIRONMENT}/github_app_installation_id`;
-    process.env.PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME = installationIdParam;
+  it('returns stored installation ID when configured for an additional app', async () => {
+    const appIdParam = `/actions-runner/${ENVIRONMENT}/additional_github_app_0_id`;
+    const appKeyParam = `/actions-runner/${ENVIRONMENT}/additional_github_app_0_key_base64`;
+    const installationIdParam = `/actions-runner/${ENVIRONMENT}/additional_github_app_0_installation_id`;
+    process.env.PARAMETER_GITHUB_APPS_MANIFEST_NAME = `/actions-runner/${ENVIRONMENT}/additional_github_apps_manifest`;
+    mockedGetParameter.mockResolvedValueOnce(
+      JSON.stringify([
+        { idParamName: appIdParam, keyParamName: appKeyParam, installationIdParamName: installationIdParam },
+      ]),
+    );
     mockedGetParameters.mockResolvedValueOnce(
       new Map([
         [PARAMETER_GITHUB_APP_ID_NAME, GITHUB_APP_ID],
         [PARAMETER_GITHUB_APP_KEY_BASE64_NAME, b64],
+        [appIdParam, '2'],
+        [appKeyParam, b64],
         [installationIdParam, '12345'],
       ]),
     );
 
-    const result = await getStoredInstallationId(0);
+    const result = await getStoredInstallationId(1);
     expect(result).toBe(12345);
   });
 
-  it('returns undefined when installation ID param is empty', async () => {
-    process.env.PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME = '';
+  it('returns undefined when the manifest env var is empty', async () => {
+    process.env.PARAMETER_GITHUB_APPS_MANIFEST_NAME = '';
     mockedGetParameters.mockResolvedValueOnce(
       new Map([
         [PARAMETER_GITHUB_APP_ID_NAME, GITHUB_APP_ID],
@@ -369,8 +379,8 @@ describe('Test getStoredInstallationId', () => {
     expect(result).toBeUndefined();
   });
 
-  it('returns undefined when env var is not set', async () => {
-    delete process.env.PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME;
+  it('returns undefined when the manifest env var is not set', async () => {
+    delete process.env.PARAMETER_GITHUB_APPS_MANIFEST_NAME;
     mockedGetParameters.mockResolvedValueOnce(
       new Map([
         [PARAMETER_GITHUB_APP_ID_NAME, GITHUB_APP_ID],
@@ -383,7 +393,7 @@ describe('Test getStoredInstallationId', () => {
   });
 
   it('returns undefined for out-of-bounds appIndex', async () => {
-    process.env.PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME = '';
+    delete process.env.PARAMETER_GITHUB_APPS_MANIFEST_NAME;
     mockedGetParameters.mockResolvedValueOnce(
       new Map([
         [PARAMETER_GITHUB_APP_ID_NAME, GITHUB_APP_ID],
@@ -395,21 +405,21 @@ describe('Test getStoredInstallationId', () => {
     expect(result).toBeUndefined();
   });
 
-  it('loads installation IDs for multi-app setup', async () => {
-    const app1IdParam = `/actions-runner/${ENVIRONMENT}/github_app_id`;
+  it('loads installation IDs for multi-app setup from the manifest', async () => {
     const app2IdParam = `/actions-runner/${ENVIRONMENT}/additional_github_app_0_id`;
-    const app1KeyParam = `/actions-runner/${ENVIRONMENT}/github_app_key_base64`;
     const app2KeyParam = `/actions-runner/${ENVIRONMENT}/additional_github_app_0_key_base64`;
     const app2InstallParam = `/actions-runner/${ENVIRONMENT}/additional_github_app_0_installation_id`;
 
-    process.env.PARAMETER_GITHUB_APP_ID_NAME = `${app1IdParam}:${app2IdParam}`;
-    process.env.PARAMETER_GITHUB_APP_KEY_BASE64_NAME = `${app1KeyParam}:${app2KeyParam}`;
-    process.env.PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME = `:${app2InstallParam}`;
-
+    process.env.PARAMETER_GITHUB_APPS_MANIFEST_NAME = `/actions-runner/${ENVIRONMENT}/additional_github_apps_manifest`;
+    mockedGetParameter.mockResolvedValueOnce(
+      JSON.stringify([
+        { idParamName: app2IdParam, keyParamName: app2KeyParam, installationIdParamName: app2InstallParam },
+      ]),
+    );
     mockedGetParameters.mockResolvedValueOnce(
       new Map([
-        [app1IdParam, '1'],
-        [app1KeyParam, b64],
+        [PARAMETER_GITHUB_APP_ID_NAME, '1'],
+        [PARAMETER_GITHUB_APP_KEY_BASE64_NAME, b64],
         [app2IdParam, '2'],
         [app2KeyParam, b64],
         [app2InstallParam, '67890'],

lambdas/functions/control-plane/src/modules.d.ts

@@ -15,6 +15,7 @@ declare namespace NodeJS {
     PARAMETER_GITHUB_APP_CLIENT_SECRET_NAME: string;
     PARAMETER_GITHUB_APP_ID_NAME: string;
     PARAMETER_GITHUB_APP_KEY_BASE64_NAME: string;
+    PARAMETER_GITHUB_APPS_MANIFEST_NAME?: string;
     RUNNER_OWNER: string;
     COMPUTE_PROVIDER_TYPE?: string;
     SCALE_DOWN_CONFIG: string;

lambdas/libs/storage-providers/aws/ssm/environment.d.ts

@@ -9,6 +9,7 @@ declare global {
       PARAMETER_GITHUB_APP_ID_NAME?: string;
       PARAMETER_GITHUB_APP_KEY_BASE64_NAME?: string;
       PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME?: string;
+      PARAMETER_GITHUB_APPS_MANIFEST_NAME?: string;
     }
   }
 }

lambdas/libs/storage-providers/aws/ssm/github-app-credentials-store.test.ts

@@ -1,10 +1,11 @@
 import { beforeEach, describe, expect, it, vi } from 'vitest';
 
-import { getParameters } from '@aws-github-runner/aws-ssm-util';
+import { getParameter, getParameters } from '@aws-github-runner/aws-ssm-util';
 
 import { createAwsSsmGitHubAppCredentialsStore } from './github-app-credentials-store';
 
 vi.mock('@aws-github-runner/aws-ssm-util', () => ({
+  getParameter: vi.fn(),
   getParameters: vi.fn(),
 }));
 
@@ -28,7 +29,7 @@ describe('aws_ssm GitHub App credentials store', () => {
     vi.clearAllMocks();
     process.env.PARAMETER_GITHUB_APP_ID_NAME = 'app-id';
     process.env.PARAMETER_GITHUB_APP_KEY_BASE64_NAME = 'app-key';
-    delete process.env.PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME;
+    delete process.env.PARAMETER_GITHUB_APPS_MANIFEST_NAME;
   });
 
   it('loads batched credentials and decodes escaped newlines', async () => {
@@ -47,9 +48,12 @@ describe('aws_ssm GitHub App credentials store', () => {
   });
 
   it('loads per-app installation IDs in the same order as app IDs', async () => {
-    process.env.PARAMETER_GITHUB_APP_ID_NAME = 'id-0:id-1';
-    process.env.PARAMETER_GITHUB_APP_KEY_BASE64_NAME = 'key-0:key-1';
-    process.env.PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME = ':installation-1';
+    process.env.PARAMETER_GITHUB_APP_ID_NAME = 'id-0';
+    process.env.PARAMETER_GITHUB_APP_KEY_BASE64_NAME = 'key-0';
+    process.env.PARAMETER_GITHUB_APPS_MANIFEST_NAME = 'manifest';
+    vi.mocked(getParameter).mockResolvedValue(
+      JSON.stringify([{ idParamName: 'id-1', keyParamName: 'key-1', installationIdParamName: 'installation-1' }]),
+    );
     getParametersMock.mockResolvedValue(
       new Map([
         ['id-0', '123'],
@@ -71,9 +75,11 @@ describe('aws_ssm GitHub App credentials store', () => {
     expect(() => createAwsSsmGitHubAppCredentialsStore()).toThrow(`Environment variable ${name} is not set`);
   });
 
-  it('rejects mismatched app and key parameter lists', () => {
-    process.env.PARAMETER_GITHUB_APP_ID_NAME = 'id-0:id-1';
-    expect(() => createAwsSsmGitHubAppCredentialsStore()).toThrow('parameter count mismatch');
+  it('rejects malformed manifest JSON', async () => {
+    process.env.PARAMETER_GITHUB_APPS_MANIFEST_NAME = 'manifest';
+    vi.mocked(getParameter).mockResolvedValue('invalid-json');
+    await expect(createAwsSsmGitHubAppCredentialsStore().get()).rejects.toThrow();
+    expect(getParametersMock).not.toHaveBeenCalled();
   });
 
   it('logs safe context when a credential parameter is missing', async () => {

lambdas/libs/storage-providers/aws/ssm/github-app-credentials-store.ts

@@ -1,4 +1,4 @@
-import { getParameters } from '@aws-github-runner/aws-ssm-util';
+import { getParameter, getParameters } from '@aws-github-runner/aws-ssm-util';
 
 import type { GitHubAppCredential, GitHubAppCredentialsStore } from '../../core';
 import { createAwsSsmStorageLogger, getErrorNames } from './logger';
@@ -8,42 +8,55 @@ const logger = createAwsSsmStorageLogger('github-app-credentials-store');
 interface AwsSsmGitHubAppCredentialsEnvironment {
   PARAMETER_GITHUB_APP_ID_NAME?: string;
   PARAMETER_GITHUB_APP_KEY_BASE64_NAME?: string;
-  PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME?: string;
+  PARAMETER_GITHUB_APPS_MANIFEST_NAME?: string;
 }
 
 export function createAwsSsmGitHubAppCredentialsStore(
   environment: Readonly<AwsSsmGitHubAppCredentialsEnvironment> = process.env,
 ): GitHubAppCredentialsStore {
-  const idParameters = splitParameterNames(environment.PARAMETER_GITHUB_APP_ID_NAME, 'PARAMETER_GITHUB_APP_ID_NAME');
-  const keyParameters = splitParameterNames(
+  const idParameter = requireParameterName(environment.PARAMETER_GITHUB_APP_ID_NAME, 'PARAMETER_GITHUB_APP_ID_NAME');
+  const keyParameter = requireParameterName(
     environment.PARAMETER_GITHUB_APP_KEY_BASE64_NAME,
     'PARAMETER_GITHUB_APP_KEY_BASE64_NAME',
   );
-  const installationIdParameters = environment.PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME?.split(':') ?? [];
-
-  if (idParameters.length !== keyParameters.length) {
-    throw new Error(`GitHub App parameter count mismatch: ${idParameters.length} IDs vs ${keyParameters.length} keys`);
-  }
+  return new AwsSsmGitHubAppCredentialsStore(
+    idParameter,
+    keyParameter,
+    environment.PARAMETER_GITHUB_APPS_MANIFEST_NAME,
+  );
+}
 
-  return new AwsSsmGitHubAppCredentialsStore(idParameters, keyParameters, installationIdParameters);
+interface AdditionalAppManifestEntry {
+  idParamName: string;
+  keyParamName: string;
+  installationIdParamName?: string | null;
 }
 
 class AwsSsmGitHubAppCredentialsStore implements GitHubAppCredentialsStore {
   constructor(
-    private readonly idParameters: string[],
-    private readonly keyParameters: string[],
-    private readonly installationIdParameters: string[],
+    private readonly idParameter: string,
+    private readonly keyParameter: string,
+    private readonly manifestParameter?: string,
   ) {}
 
   async get(): Promise<GitHubAppCredential[]> {
+    const entries: AdditionalAppManifestEntry[] = [{ idParamName: this.idParameter, keyParamName: this.keyParameter }];
+    if (this.manifestParameter) {
+      const manifest = JSON.parse(await getParameter(this.manifestParameter)) as AdditionalAppManifestEntry[];
+      entries.push(...manifest);
+    }
+    const idParameters = entries.map((entry) => entry.idParamName);
+    const keyParameters = entries.map((entry) => entry.keyParamName);
+    const installationIdParameters = entries.map((entry) => entry.installationIdParamName);
+
     const parameterNames = [
-      ...this.idParameters,
-      ...this.keyParameters,
-      ...this.installationIdParameters.filter(Boolean),
+      ...idParameters,
+      ...keyParameters,
+      ...installationIdParameters.filter((name): name is string => Boolean(name)),
     ];
     logger.debug('Reading GitHub App credential parameters', {
       parameterCount: parameterNames.length,
-      appCount: this.idParameters.length,
+      appCount: idParameters.length,
     });
 
     let parameters: Map<string, string>;
@@ -52,13 +65,13 @@ class AwsSsmGitHubAppCredentialsStore implements GitHubAppCredentialsStore {
     } catch (error) {
       logger.error('Failed to read GitHub App credential parameters', {
         parameterCount: parameterNames.length,
-        appCount: this.idParameters.length,
+        appCount: idParameters.length,
         errorNames: getErrorNames(error),
       });
       throw error;
     }
 
-    const credentials = this.idParameters.map((idParameter, index) => {
+    const credentials = idParameters.map((idParameter, index) => {
       const appIdValue = parameters.get(idParameter);
       if (!appIdValue) {
         logger.error('GitHub App credential parameter is missing', {
@@ -68,7 +81,7 @@ class AwsSsmGitHubAppCredentialsStore implements GitHubAppCredentialsStore {
         });
         throw new Error(`Parameter ${idParameter} not found`);
       }
-      const keyParameter = this.keyParameters[index];
+      const keyParameter = keyParameters[index];
       const privateKeyBase64 = parameters.get(keyParameter);
       if (!privateKeyBase64) {
         logger.error('GitHub App credential parameter is missing', {
@@ -78,7 +91,7 @@ class AwsSsmGitHubAppCredentialsStore implements GitHubAppCredentialsStore {
         });
         throw new Error(`Parameter ${keyParameter} not found`);
       }
-      const installationIdParameter = this.installationIdParameters[index];
+      const installationIdParameter = installationIdParameters[index];
       const installationIdValue = installationIdParameter ? parameters.get(installationIdParameter) : undefined;
       return {
         appId: Number.parseInt(appIdValue, 10),
@@ -95,9 +108,9 @@ class AwsSsmGitHubAppCredentialsStore implements GitHubAppCredentialsStore {
   }
 }
 
-function splitParameterNames(value: string | undefined, name: string): string[] {
+function requireParameterName(value: string | undefined, name: string): string {
   if (!value || value.trim() === '') {
     throw new Error(`Environment variable ${name} is not set`);
   }
-  return value.split(':').filter(Boolean);
+  return value;
 }

main.tf

@@ -7,19 +7,18 @@ locals {
   primary_app_key_base64 = coalesce(var.github_app.key_base64_ssm, module.ssm.parameters.github_app_key_base64)
 
   github_app_parameters = {
-    id = concat(
-      [local.primary_app_id],
-      [for p in module.ssm.additional_app_parameters : p.id]
-    )
-    key_base64 = concat(
-      [local.primary_app_key_base64],
-      [for p in module.ssm.additional_app_parameters : p.key_base64]
-    )
-    installation_id = concat(
-      [null],
-      [for p in module.ssm.additional_app_parameters : p.installation_id]
-    )
+    id             = local.primary_app_id
+    key_base64     = local.primary_app_key_base64
     webhook_secret = coalesce(var.github_app.webhook_secret_ssm, module.ssm.parameters.github_app_webhook_secret)
+    # Additional apps flow to the lambdas through the manifest parameter so
+    # the lambda environment size stays constant regardless of app count.
+    additional_apps_manifest = module.ssm.additional_apps_manifest
+    additional_app_parameter_arns = flatten([
+      for p in module.ssm.additional_app_parameters : concat(
+        [p.id.arn, p.key_base64.arn],
+        p.installation_id != null ? [p.installation_id.arn] : []
+      )
+    ])
   }
 
   default_runner_labels = distinct(concat(["self-hosted", var.runner_os, var.runner_architecture]))
@@ -406,8 +405,8 @@ locals {
     metrics                      = var.metrics
     enable_runner_deregistration = var.instance_termination_watcher.enable_runner_deregistration
     github_app_parameters = var.instance_termination_watcher.enable_runner_deregistration ? {
-      id         = local.github_app_parameters.id[0]
-      key_base64 = local.github_app_parameters.key_base64[0]
+      id         = local.github_app_parameters.id
+      key_base64 = local.github_app_parameters.key_base64
     } : null
     ghes_url = var.ghes_url
   }

modules/multi-runner/main.tf

@@ -7,19 +7,18 @@ locals {
   primary_app_key_base64 = coalesce(local.effective_config.github.app.key_base64_ssm, module.ssm.parameters.github_app_key_base64)
 
   github_app_parameters = {
-    id = concat(
-      [local.primary_app_id],
-      [for p in module.ssm.additional_app_parameters : p.id]
-    )
-    key_base64 = concat(
-      [local.primary_app_key_base64],
-      [for p in module.ssm.additional_app_parameters : p.key_base64]
-    )
-    installation_id = concat(
-      [null],
-      [for p in module.ssm.additional_app_parameters : p.installation_id]
-    )
+    id             = local.primary_app_id
+    key_base64     = local.primary_app_key_base64
     webhook_secret = coalesce(local.effective_config.github.app.webhook_secret_ssm, module.ssm.parameters.github_app_webhook_secret)
+    # Additional apps flow to the lambdas through the manifest parameter so
+    # the lambda environment size stays constant regardless of app count.
+    additional_apps_manifest = module.ssm.additional_apps_manifest
+    additional_app_parameter_arns = flatten([
+      for p in module.ssm.additional_app_parameters : concat(
+        [p.id.arn, p.key_base64.arn],
+        p.installation_id != null ? [p.installation_id.arn] : []
+      )
+    ])
   }
 
   ssm_root_path = trimsuffix(coalesce(