feat(terraform): wire shared DynamoDB runner storage

.github/workflows/terraform.yml

@@ -86,6 +86,13 @@ jobs:
             "lambda",
             "multi-runner",
             "runner-binaries-syncer",
+            "storage-providers/aws/dynamodb",
+            "orchestration-providers/webhook",
+            "orchestration-providers/webhook/job-retry",
+            "orchestration-providers/webhook/pool",
+            "orchestration-providers/webhook/scale-runners",
+            "compute-providers/aws/ec2",
+            "compute-providers/aws/ec2/trust-policy",
             "runners",
             "setup-iam-permissions",
             "ssm",
@@ -215,6 +222,13 @@ jobs:
         module:
           - modules/runners
           - modules/multi-runner
+          - modules/orchestration-providers/webhook
+          - modules/orchestration-providers/webhook/job-retry
+          - modules/orchestration-providers/webhook/pool
+          - modules/orchestration-providers/webhook/scale-runners
+          - modules/storage-providers/aws/dynamodb
+          - modules/compute-providers/aws/ec2
+          - modules/compute-providers/aws/ec2/trust-policy
     defaults:
       run:
         working-directory: ${{ matrix.module }}

docs/adr/0002-runner-storage-provider-boundary.md

@@ -0,0 +1,196 @@
+# ADR-0002: Runner Storage Provider Boundary
+
+## Status
+
+Proposed
+
+## Date
+
+2026-09-08
+
+## Context
+
+Runner operation depends on several kinds of stored data: GitHub App
+credentials, webhook secrets, matcher configuration, runner-group mappings,
+short-lived runner bootstrap configuration, and runner lifecycle state. The
+original implementation stored these values in AWS Systems Manager Parameter
+Store (SSM), with parameter names, SecureString handling, cleanup, and IAM
+permissions spread across the Lambda and Terraform modules.
+
+That coupling makes it difficult to provide durable runner inventory and to
+change the storage implementation without adding provider-specific branches to
+each consumer. It also makes the control plane rely on provider discovery for
+runner counts, which is insufficient while a runner is being provisioned or
+when a launch succeeds before the rest of its registration flow completes.
+
+The repository needs a replaceable storage boundary that preserves existing
+SSM deployments while allowing an opt-in DynamoDB implementation for the
+provider-boundary multi-runner configuration.
+
+## Decision
+
+Define provider-neutral storage interfaces for the data used by runner
+orchestration and select one storage provider for a deployment. The supported
+providers are:
+
+- `aws_ssm`: the existing default and compatibility path.
+- `aws_dynamodb`: the durable, opt-in path for provider-boundary
+  multi-runner configurations.
+
+Provider selection is represented by the canonical values `aws_ssm` and
+`aws_dynamodb`. An omitted selection resolves to `aws_ssm`. A deployment must
+select at most one provider; storage consumers do not silently fall back from
+one provider to the other when a credential, permission, or data lookup fails.
+
+### Provider-neutral contract
+
+The storage library owns interfaces and provider factories for:
+
+- GitHub App credentials;
+- webhook secrets;
+- runner matcher configuration;
+- runner configuration creation and one-time consumption;
+- runner-group ID caching; and
+- runner lifecycle state.
+
+Control-plane and bootstrap code depends on these interfaces. It does not
+construct SSM parameter names or DynamoDB keys. Provider-specific factories are
+selected once per Lambda process from the environment and are safe to reuse
+within that process.
+
+Runner bootstrap configuration remains separate from lifecycle state. Bootstrap
+configuration contains short-lived or sensitive values and is consumed once;
+runner state is durable inventory keyed by the compute resource and records
+states such as `provisioning`, `active`, `orphan`, and `terminating`.
+
+### SSM provider
+
+The SSM provider retains the established behavior for existing deployments:
+
+- parameters remain the storage boundary for credentials, secrets, matcher
+  configuration, runner groups, and runner bootstrap configuration;
+- sensitive values use SecureString parameters and existing parameter-store
+  tagging conventions;
+- runner configuration cleanup remains an explicit housekeeper operation; and
+- existing stable Terraform inputs continue to translate to the SSM provider.
+
+SSM does not provide the durable runner-state implementation in this phase.
+When state inventory is unavailable, the control plane uses compute-provider
+discovery, preserving the existing behavior.
+
+### DynamoDB provider
+
+The DynamoDB provider uses two shared tables:
+
+1. a configuration table for global records and per-runner-entry records; and
+2. a runner-state table for durable lifecycle inventory with TTL-based cleanup.
+
+Records use explicit logical scopes and an `id` so that global data, entry
+configuration, runner-group mappings, bootstrap values, and runner state cannot
+collide. The provider exposes table names, scopes, TTL settings, and IAM policy
+fragments as Terraform capabilities rather than making callers know the table
+layout.
+
+The DynamoDB implementation must enforce the storage contract at the data
+operation boundary:
+
+- one-time bootstrap consumption is conditional and removes the consumed
+  record;
+- lifecycle transitions are conditional so stale workers cannot overwrite a
+  newer state;
+- runner-state records identify the compute provider, compute resource, GitHub
+  identity when known, owner, runner type, and lifecycle state; and
+- IAM policies restrict access with table ARNs and DynamoDB leading-key
+  conditions. Runner bootstrap access is restricted to the matching compute
+  resource identity.
+
+### Terraform capability boundary
+
+Terraform resolves the selected provider once and passes opaque capabilities to
+the webhook orchestration and compute-provider modules. Capabilities include
+provider-specific environment variables and IAM policy documents for each
+consumer, including the runner bootstrap path.
+
+The `global_config_storage_provider` input selects the provider for the
+provider-boundary configuration. Stable v1 configuration is translated to an
+SSM selection, so existing users retain the current backend unless they opt in
+to DynamoDB through the provider-boundary configuration.
+
+The compute provider owns the runner-side capability needed to read bootstrap
+configuration. The orchestration provider owns its Lambda resources and
+receives only the capabilities it needs. This keeps storage ownership separate
+from both compute implementation and orchestration scheduling.
+
+## Alternatives considered
+
+### Keep SSM as the only backend
+
+This preserves the smallest implementation, but does not provide durable
+runner inventory or a suitable shared store for the provider-boundary design.
+
+### Add storage conditionals to every consumer
+
+This would avoid a factory layer initially, but it would duplicate key
+construction, error handling, security rules, and migration behavior across
+Lambdas and runner bootstrap code. It would make each new provider more
+expensive and easier to implement inconsistently.
+
+### Use one DynamoDB table for all data
+
+One table could reduce resource count, but separating configuration from
+ephemeral runner state gives the two lifecycles independent TTL, protection,
+and access policies. The two-table design also makes accidental access to
+runner state from configuration consumers less likely.
+
+### Migrate existing SSM data automatically
+
+Automatic migration would require dual writes or a cutover protocol and could
+duplicate or lose short-lived bootstrap configuration. Migration is therefore
+an explicit operational decision outside provider selection; the default
+remains backward compatible with SSM.
+
+## Consequences
+
+### Positive
+
+- Existing stable deployments continue to use SSM without configuration
+  changes.
+- Storage consumers share one provider-neutral contract and do not duplicate
+  backend logic.
+- DynamoDB can provide durable runner inventory and conservative recovery from
+  launch-before-registration failures.
+- Provider-specific IAM conditions and runner bootstrap capabilities can be
+  reviewed at the Terraform module boundary.
+- A future storage provider can implement the same interfaces without changing
+  orchestration or compute-provider callers.
+
+### Negative
+
+- The DynamoDB path adds two tables, TTL behavior, conditional-write logic,
+  provider-specific IAM, and additional operational cost.
+- SSM and DynamoDB have different consistency, cleanup, and failure behavior;
+  both implementations require provider-specific contract tests.
+- Switching an existing deployment does not migrate stored values or active
+  runner inventory automatically.
+- The control plane must retain compute discovery as a recovery source even
+  when DynamoDB inventory is enabled.
+
+## Migration and operational rules
+
+1. Keep `aws_ssm` as the default until a deployment explicitly selects
+   `aws_dynamodb`.
+2. Treat a provider switch as an operational migration with a planned cutover;
+   do not assume existing SSM records are present in DynamoDB.
+3. Keep provider-specific secrets, table names, scopes, and IAM details inside
+   provider capabilities and environment configuration, not in shared
+   orchestration code.
+4. Add contract tests for every new provider covering reads, writes,
+   one-time consumption, conditional lifecycle transitions, and authorization
+   boundaries.
+
+## References
+
+- [Storage-provider interfaces and factories](../../lambdas/libs/storage-providers/)
+- [DynamoDB storage-provider module](../../modules/storage-providers/aws/dynamodb/)
+- [Multi-runner storage-provider composition](../../modules/multi-runner/storage-provider.tf)
+- [Compute-provider storage capability contract](../../modules/compute-providers/aws/ec2/variables.tf)

mkdocs.yaml

@@ -60,6 +60,7 @@ nav:
   - Security: security.md
   - Architecture decisions:
     - MiniStack for integration tests: adr/0001-use-ministack-for-terraform-integration-tests.md
+    - Runner storage provider boundary: adr/0002-runner-storage-provider-boundary.md
   - Modules:
     - Runners (main): modules/runners.md
     - Submodules (public):

modules/compute-providers/aws/ec2/README.md

@@ -67,6 +67,7 @@ No modules.
 | <a name="input_prefix"></a> [prefix](#input\_prefix) | Prefix used to identify resources created for the runner configuration. | `string` | `"github-actions"` | no |
 | <a name="input_runner"></a> [runner](#input\_runner) | Provider-neutral runner settings consumed by compute providers.<br/><br/>- `os`: Runner operating system. Supported values are `linux`, `osx`, and `windows`.<br/>- `architecture`: Runner distribution architecture.<br/>- `name_prefix`: Prefix added to registered runner names.<br/>- `run_as_root`: Runs the runner service as root.<br/>- `run_as`: Operating-system user used when `run_as_root` is false.<br/>- `hooks.job_started`: Script installed as the runner job-started hook.<br/>- `hooks.job_completed`: Script installed as the runner job-completed hook.<br/>- `iam.role.arn`: Resolved runner-role ARN referenced by provider policies and resources.<br/>- `iam.role.name`: Resolved runner-role name used by provider resources.<br/>- `iam.role.managed`: Whether runner-config manages the resolved runner role.<br/>- `iam.managed_policy_arns`: Common managed-policy ARNs returned with the provider-specific runner policies for attachment by runner-config.<br/>- `iam.path`: IAM path available to provider-managed IAM resources. Null derives the path from `prefix`. | <pre>object({<br/>    os           = optional(string, "linux")<br/>    architecture = optional(string, "x64")<br/>    name_prefix  = optional(string, "")<br/>    run_as_root  = optional(bool, false)<br/>    run_as       = optional(string, "ec2-user")<br/>    hooks = optional(object({<br/>      job_started   = optional(string, "")<br/>      job_completed = optional(string, "")<br/>    }), {})<br/>    iam = object({<br/>      role = object({<br/>        arn     = string<br/>        name    = string<br/>        managed = optional(bool, true)<br/>      })<br/>      managed_policy_arns = optional(map(string), {})<br/>      path                = optional(string, null)<br/>    })<br/>  })</pre> | n/a | yes |
 | <a name="input_ssm"></a> [ssm](#input\_ssm) | Parameter Store paths and tag scopes available to compute-provider bootstrap resources.<br/><br/>- `paths.root`: Root Parameter Store path for the runner configuration.<br/>- `paths.tokens`: Path segment used for registration tokens and just-in-time configuration.<br/>- `paths.config`: Path segment used for persistent runner and provider configuration.<br/>- `tags`: Shared SSM tags that override module-level `tags`.<br/>- `parameters.tags`: Parameter-specific tags that override module-level and shared SSM tags. | <pre>object({<br/>    paths = object({<br/>      root   = string<br/>      tokens = string<br/>      config = string<br/>    })<br/>    tags = optional(map(string), {})<br/>    parameters = optional(object({<br/>      tags = optional(map(string), {})<br/>    }), {})<br/>  })</pre> | n/a | yes |
+| <a name="input_storage_provider"></a> [storage\_provider](#input\_storage\_provider) | Runner-side storage locator and opaque IAM policy supplied by runner-config. The default preserves the existing SSM bootstrap path. | <pre>object({<br/>    type = string<br/>    runner = object({<br/>      config_table_name       = optional(string, null)<br/>      runner_state_table_name = optional(string, null)<br/>      scope                   = optional(string, null)<br/>      iam_policy_json         = optional(string, null)<br/>    })<br/>  })</pre> | <pre>{<br/>  "runner": {<br/>    "config_table_name": null,<br/>    "iam_policy_json": null,<br/>    "runner_state_table_name": null,<br/>    "scope": null<br/>  },<br/>  "type": "aws_ssm"<br/>}</pre> | no |
 | <a name="input_tags"></a> [tags](#input\_tags) | Base tags available to taggable compute-provider resources. Provider-specific tags override this map within their documented scopes. | `map(string)` | `{}` | no |
 
 ## Outputs

modules/compute-providers/aws/ec2/control-plane.tf

@@ -191,7 +191,7 @@ data "aws_iam_policy_document" "service_linked_role" {
 }
 
 locals {
-  scale_up_environment_variables = {
+  scale_up_environment_variables = merge({
     AMI_ID_SSM_PARAMETER_NAME            = local.ami_id_ssm_parameter_name
     INSTANCE_ALLOCATION_STRATEGY         = var.config.instance_allocation_strategy
     INSTANCE_MAX_SPOT_PRICE              = var.config.instance_max_spot_price
@@ -203,7 +203,9 @@ locals {
     ENABLE_ON_DEMAND_FAILOVER_FOR_ERRORS = jsonencode(var.config.on_demand_failover_for_errors)
     SCALE_ERRORS                         = jsonencode(var.config.scale_errors)
     USE_DEDICATED_HOST                   = var.config.use_dedicated_host
-  }
+    }, var.storage_provider.type == "aws_dynamodb" ? {
+    EC2_INSTANCE_ARN_PREFIX = local.ec2_instance_arn_prefix
+  } : {})
 
   scale_down_environment_variables = {}
 

modules/compute-providers/aws/ec2/policies-runner.tf

@@ -4,6 +4,7 @@ data "aws_caller_identity" "current" {}
 
 locals {
   ssm_parameter_arn_prefix = "arn:${var.aws_partition}:ssm:${var.aws_region}:${data.aws_caller_identity.current.account_id}:parameter"
+  ec2_instance_arn_prefix  = "arn:${var.aws_partition}:ec2:${var.aws_region}:${data.aws_caller_identity.current.account_id}:instance/"
   ssm_config_arn           = "${local.ssm_parameter_arn_prefix}${var.ssm.paths.root}/${var.ssm.paths.config}"
   cloudwatch_config_arn    = "${local.ssm_config_arn}/cloudwatch_agent_config_runner"
 }
@@ -167,10 +168,6 @@ data "aws_iam_policy_document" "cloudwatch" {
 locals {
   runner_inline_policies = merge(
     {
-      ssm_parameters = {
-        name        = "runner-ssm-parameters"
-        policy_json = data.aws_iam_policy_document.ssm_parameters.json
-      }
       describe_tags = {
         name        = "runner-describe-tags"
         policy_json = data.aws_iam_policy_document.describe_tags.json
@@ -184,6 +181,17 @@ locals {
         policy_json = data.aws_iam_policy_document.terminate_self.json
       }
     },
+    var.storage_provider.type == "aws_ssm" ? {
+      ssm_parameters = {
+        name        = "runner-ssm-parameters"
+        policy_json = data.aws_iam_policy_document.ssm_parameters.json
+      }
+      } : {
+      runner_config_storage = {
+        name        = "runner-config-storage"
+        policy_json = var.storage_provider.runner.iam_policy_json
+      }
+    },
     var.config.ssm_enabled ? {
       session_manager = {
         name        = "runner-ssm-session"

modules/compute-providers/aws/ec2/runner-config.tf

@@ -1,13 +1,25 @@
 resource "aws_ssm_parameter" "runner_config_run_as" {
+  count = var.storage_provider.type == "aws_ssm" ? 1 : 0
   name  = "${var.ssm.paths.root}/${var.ssm.paths.config}/run_as"
   type  = "String"
   value = var.runner.run_as_root ? "root" : var.runner.run_as
   tags  = local.ssm_parameter_tags
 }
 
 resource "aws_ssm_parameter" "runner_enable_cloudwatch" {
+  count = var.storage_provider.type == "aws_ssm" ? 1 : 0
   name  = "${var.ssm.paths.root}/${var.ssm.paths.config}/enable_cloudwatch"
   type  = "String"
   value = var.config.cloudwatch_agent.enabled
   tags  = local.ssm_parameter_tags
 }
+
+moved {
+  from = aws_ssm_parameter.runner_config_run_as
+  to   = aws_ssm_parameter.runner_config_run_as[0]
+}
+
+moved {
+  from = aws_ssm_parameter.runner_enable_cloudwatch
+  to   = aws_ssm_parameter.runner_enable_cloudwatch[0]
+}

modules/compute-providers/aws/ec2/runner-instances.tf

@@ -90,7 +90,12 @@ locals {
     hook_job_started   = var.runner.hooks.job_started
     hook_job_completed = var.runner.hooks.job_completed
     start_runner = templatefile(local.userdata_start_runner[var.runner.os], {
-      metadata_tags = var.config.metadata_options != null ? var.config.metadata_options.instance_metadata_tags : "enabled"
+      metadata_tags                     = var.config.metadata_options != null ? var.config.metadata_options.instance_metadata_tags : "enabled"
+      storage_provider_type             = var.storage_provider.type
+      dynamodb_config_table_name_base64 = var.storage_provider.type == "aws_dynamodb" ? base64encode(var.storage_provider.runner.config_table_name) : ""
+      dynamodb_scope_base64             = var.storage_provider.type == "aws_dynamodb" ? base64encode(var.storage_provider.runner.scope) : ""
+      ec2_instance_arn_prefix_base64    = var.storage_provider.type == "aws_dynamodb" ? base64encode(local.ec2_instance_arn_prefix) : ""
+      enable_cloudwatch_agent           = var.config.cloudwatch_agent.enabled
     })
     ghes_url        = var.github.enterprise_server.url
     ghes_ssl_verify = var.github.enterprise_server.ssl_verify