diff --git a/.github/workflows/ministack.yml b/.github/workflows/ministack.yml index 3f8b7c6b38..3ee7436d69 100644 --- a/.github/workflows/ministack.yml +++ b/.github/workflows/ministack.yml @@ -11,6 +11,7 @@ on: - "policies/**" - "examples/**" - "modules/**" + - "lambdas/**" pull_request: paths: - ".github/workflows/ministack.yml" @@ -19,6 +20,7 @@ on: - "policies/**" - "examples/**" - "modules/**" + - "lambdas/**" workflow_dispatch: concurrency: @@ -104,3 +106,64 @@ jobs: env: EXAMPLE: ${{ matrix.example }} run: tests/ministack/run-example.sh destroy "$EXAMPLE" + + integration_smoke: + name: Run webhook and pool lifecycle smoke test against MiniStack + runs-on: ubuntu-latest + timeout-minutes: 30 + services: + ministack: + image: ghcr.io/ministackorg/ministack:1.5.7@sha256:37361b9ef886463d5632d5a4b2d114da4b7a5c5793f52f07dbc72579f2fd9207 + ports: + - 4566:4566 + options: --add-host=host.docker.internal:host-gateway + env: + MINISTACK_ACCOUNT_ID: "000000000000" + MINISTACK_REGION: eu-west-1 + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1 + with: + egress-policy: audit + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: lambdas/.nvmrc + package-manager-cache: false + + - name: Setup Terraform + uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 + with: + terraform_version: latest + terraform_wrapper: false + + - name: Install Lambda dependencies + working-directory: lambdas + run: yarn install --frozen-lockfile + + - name: Build smoke-test Lambda distributions + working-directory: lambdas + run: | + yarn workspace @aws-github-runner/webhook dist + yarn workspace @aws-github-runner/control-plane dist + + - name: Start MockServer + id: mockserver + uses: mock-server/setup-mockserver@24612c2ccef1f83d587f331ed77cc5cef441e0b1 # v1.0.0 + with: + version: '7.6.0@sha256:80b3b1a26f3553d0c81a3f3896b5b7274c17b2a2e52f0fd2b28e246bc9efa290' + port: '1080' + startup-timeout: '60' + + - name: Run webhook and pool lifecycle smoke test + env: + MINISTACK_GITHUB_MOCK_HOST: host.docker.internal + MINISTACK_GITHUB_MOCK_PORT: "1080" + MINISTACK_GITHUB_MOCK_URL: ${{ steps.mockserver.outputs.url }} + run: sh tests/ministack/run-smoke.sh diff --git a/tests/ministack/README.md b/tests/ministack/README.md index a64f44f134..ec42cf2f8f 100644 --- a/tests/ministack/README.md +++ b/tests/ministack/README.md @@ -36,3 +36,52 @@ MiniStack's AWS-compatible EC2 API, then removes only the resources it created during cleanup. MiniStack v1.5.7 provides the EC2 image behavior needed by the `default`, `ephemeral`, and `multi-runner` examples, so they are included in the same lifecycle matrix. + +## Webhook and runner lifecycle smoke test + +The smoke test covers two independent lifecycle chains. The webhook chain +sends signed `workflow_job` webhooks through the API Gateway endpoint and +verifies the asynchronous path through EventBridge, the dispatcher Lambda, SQS, +and the scale-up Lambda. It runs scale-up once without a dynamic label and once +with `ghr-ec2-instance-type:m5.large`, checking that the first launch uses a +configured default instance type and the second launch uses exactly `m5.large`. +The scale-up Lambda calls a pinned `mockserver/mockserver` container initialized +from `github-api-expectations.json`; the test uses MockServer's verification API +to confirm the expected GitHub API calls for both jobs. It also checks the +webhook, dispatcher, and scale-up Lambda log groups for each smoke job ID, then +confirms that both MiniStack EC2 runner instances are removed and terminated. + +The second, pool chain then invokes the pool Lambda with a pool size of one and verifies every +expected GitHub API route for pool reconciliation, including the installation, +token, runner-list, and registration-token calls, before confirming that it +creates a second EC2 runner. Installation lookup is mocked for configurations +that do not provide a stored installation ID, but is conditional and is not a +required assertion. The test also verifies the `ghr:Application`, +`ghr:created_by`, `ghr:Type`, and `ghr:Owner` tags used to discover managed +instances. MiniStack does not currently propagate the Terraform launch-template +`ghr:environment` tag to instances, so that tag is not asserted by this smoke +test. The scale-down portions of the smoke test are temporarily commented out +because of a MiniStack `CreateFleet` issue. The assertions remain in the script +and should be re-enabled once that MiniStack behavior is fixed. +The pool schedule is configured for a far-future date because the test invokes +the Lambda directly. + +Build the two real Lambda distributions, start MiniStack, and run: + +```sh +(cd lambdas && yarn install --frozen-lockfile) +(cd lambdas && yarn workspace @aws-github-runner/webhook dist) +(cd lambdas && yarn workspace @aws-github-runner/control-plane dist) +sh tests/ministack/run-smoke.sh +``` + +The smoke script generates a temporary RSA key and Terraform variables file, +starts the MockServer container on a temporary port, and removes all temporary +state during cleanup. In CI, the pinned MockServer setup action starts the +server and waits for readiness; the expectations are loaded after checkout. +MiniStack must be able to reach +`host.docker.internal`; +override the hostname with `MINISTACK_GITHUB_MOCK_HOST` when using a different +container runtime. When MiniStack is exposed on a non-default local port, use a +host address reachable from its container for `AWS_ENDPOINT_URL`, for example +`AWS_ENDPOINT_URL=http://:14568`, instead of `127.0.0.1`. diff --git a/tests/ministack/github-api-expectations.json b/tests/ministack/github-api-expectations.json new file mode 100644 index 0000000000..a85e727bd2 --- /dev/null +++ b/tests/ministack/github-api-expectations.json @@ -0,0 +1,77 @@ +[ + { + "httpRequest": { + "method": "GET", + "path": "/api/v3/repos/test-owner/test-repo/actions/jobs/123456" + }, + "httpResponse": { + "statusCode": 200, + "headers": { + "Content-Type": ["application/json"], + "X-RateLimit-Limit": ["5000"], + "X-RateLimit-Remaining": ["4999"] + }, + "body": "{\"id\":123456,\"status\":\"queued\",\"name\":\"ministack-smoke\"}" + } + }, + { + "httpRequest": { + "method": "GET", + "path": "/api/v3/repos/test-owner/test-repo/actions/jobs/123457" + }, + "httpResponse": { + "statusCode": 200, + "headers": { + "Content-Type": ["application/json"], + "X-RateLimit-Limit": ["5000"], + "X-RateLimit-Remaining": ["4999"] + }, + "body": "{\"id\":123457,\"status\":\"queued\",\"name\":\"ministack-smoke-dynamic\"}" + } + }, + { + "httpRequest": { + "method": "POST", + "path": "/api/v3/app/installations/123/access_tokens" + }, + "httpResponse": { + "statusCode": 201, + "headers": { + "Content-Type": ["application/json"], + "X-RateLimit-Limit": ["5000"], + "X-RateLimit-Remaining": ["4999"] + }, + "body": "{\"token\":\"ministack-installation-token\",\"expires_at\":\"2099-01-01T00:00:00Z\"}" + } + }, + { + "httpRequest": { + "method": "GET", + "path": "/api/v3/orgs/test-owner/installation" + }, + "httpResponse": { + "statusCode": 200, + "headers": { + "Content-Type": ["application/json"], + "X-RateLimit-Limit": ["5000"], + "X-RateLimit-Remaining": ["4999"] + }, + "body": "{\"id\":123}" + } + }, + { + "httpRequest": { + "method": "POST", + "path": "/api/v3/orgs/test-owner/actions/runners/registration-token" + }, + "httpResponse": { + "statusCode": 201, + "headers": { + "Content-Type": ["application/json"], + "X-RateLimit-Limit": ["5000"], + "X-RateLimit-Remaining": ["4999"] + }, + "body": "{\"token\":\"ministack-registration-token\",\"expires_at\":\"2099-01-01T00:00:00Z\"}" + } + } +] diff --git a/tests/ministack/run-smoke.sh b/tests/ministack/run-smoke.sh new file mode 100644 index 0000000000..7dfa886d5e --- /dev/null +++ b/tests/ministack/run-smoke.sh @@ -0,0 +1,731 @@ +#!/bin/sh + +set -eu + +export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-000000000000}" +export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-test-only}" +export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-eu-west-1}" +export AWS_REGION="${AWS_REGION:-eu-west-1}" +export AWS_ENDPOINT_URL="${AWS_ENDPOINT_URL:-http://127.0.0.1:4566}" +export AWS_EC2_METADATA_DISABLED="${AWS_EC2_METADATA_DISABLED:-true}" + +script_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +source_root=$(CDPATH='' cd -- "$script_dir/../.." && pwd) +example_root="$source_root/examples/default" +mock_expectations="$script_dir/github-api-expectations.json" +fixture="$script_dir/workflow_job_event.json" +dynamic_fixture=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-dynamic-workflow-job.XXXXXX") +mock_host="${MINISTACK_GITHUB_MOCK_HOST:-host.docker.internal}" +mock_port="${MINISTACK_GITHUB_MOCK_PORT:-}" +mock_service_url="${MINISTACK_GITHUB_MOCK_URL:-}" +mock_image="${MINISTACK_GITHUB_MOCK_IMAGE:-mockserver/mockserver:7.6.0@sha256:80b3b1a26f3553d0c81a3f3896b5b7274c17b2a2e52f0fd2b28e246bc9efa290}" +mock_container="" +tfvars_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-smoke.XXXXXX") +app_key_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-github-app.XXXXXX") +response_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-smoke-response.XXXXXX") +lambda_response_file=$(mktemp "${TMPDIR:-/tmp}/terraform-aws-github-runner-lambda-response.XXXXXX") +override_file="$example_root/zz_ministack_smoke_override.tf" +terraform_initialized=false + +cleanup() { + set +e + if [ "$terraform_initialized" = true ]; then + "$source_root/tests/ministack/run-example.sh" destroy default "$tfvars_file" >/dev/null 2>&1 + fi + if [ -n "$mock_container" ]; then + docker rm -f "$mock_container" >/dev/null 2>&1 + fi + rm -f "$override_file" "$tfvars_file" "$app_key_file" "$response_file" "$lambda_response_file" "$dynamic_fixture" +} +trap cleanup EXIT INT TERM + +require_command() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "$1 is required to run the MiniStack smoke test." >&2 + exit 69 + fi +} + +for command in aws curl openssl python3 terraform; do + require_command "$command" +done +if [ -z "$mock_service_url" ]; then + require_command docker +fi + +for lambda_zip in \ + "$source_root/lambdas/functions/webhook/webhook.zip" \ + "$source_root/lambdas/functions/control-plane/runners.zip"; do + if [ ! -f "$lambda_zip" ]; then + echo "Missing $lambda_zip. Build the webhook and control-plane distributions first." >&2 + exit 66 + fi +done + +if [ -z "$mock_port" ]; then + if [ -n "$mock_service_url" ]; then + mock_port=1080 + else + mock_port=$(python3 -c 'import socket; s = socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()') + fi +fi + +if [ -z "$mock_service_url" ]; then + mock_container="terraform-aws-github-runner-github-api-mock-$$" + mock_service_url="http://127.0.0.1:${mock_port}" +fi + +openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "$app_key_file" 2>/dev/null +app_key_base64=$(base64 < "$app_key_file" | tr -d '\n') +APP_KEY_BASE64="$app_key_base64" python3 - "$script_dir/default.tfvars" "$tfvars_file" <<'PY' +import os +import sys + +source, destination = sys.argv[1:] +replacement = os.environ["APP_KEY_BASE64"] +with open(source, encoding="utf-8") as source_file: + lines = source_file.readlines() +with open(destination, "w", encoding="utf-8") as destination_file: + for line in lines: + if line.lstrip().startswith("key_base64 ="): + destination_file.write(f' key_base64 = "{replacement}"\n') + elif line.lstrip().startswith('id') and '=' in line: + destination_file.write(' id = "123"\n') + else: + destination_file.write(line) +PY +unset app_key_base64 APP_KEY_BASE64 + +printf '%s\n' \ + 'module "runners" {' \ + " ghes_url = \"http://${mock_host}:${mock_port}\"" \ + ' ghes_ssl_verify = false' \ + ' eventbridge = {' \ + ' enable = true' \ + ' accept_events = ["workflow_job"]' \ + ' }' \ + ' delay_webhook_event = 0' \ + ' runners_maximum_count = 1' \ + ' instance_types = ["m7a.large"]' \ + ' enable_dynamic_labels = true' \ + ' minimum_running_time_in_minutes = 0' \ + ' pool_runner_owner = "test-owner"' \ + ' pool_config = [{ schedule_expression = "cron(0 0 1 1 ? 2099)", size = 1 }]' \ + ' scale_down_schedule_expression = "cron(0 0 1 1 ? 2099)"' \ + ' enable_job_queued_check = true' \ + ' enable_jit_config = false' \ + ' enable_runner_binaries_syncer = false' \ + ' log_level = "debug"' \ + '}' \ + '' \ + 'module "webhook_github_app" {' \ + ' count = 0' \ + '}' > "$override_file" + +if [ -n "$mock_container" ]; then + docker run --detach --name "$mock_container" --publish "${mock_port}:1080" \ + --volume "$mock_expectations:/config/github-api-expectations.json:ro" \ + --env MOCKSERVER_INITIALIZATION_JSON_PATH=/config/github-api-expectations.json \ + "$mock_image" >/dev/null +fi + +attempts=30 +while ! curl -fsS --max-time 2 -X PUT "${mock_service_url}/mockserver/status" >/dev/null 2>&1; do + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "MockServer did not become ready." >&2 + if [ -n "$mock_container" ]; then + docker logs "$mock_container" >&2 + fi + exit 70 + fi + sleep 1 +done + +if [ -z "$mock_container" ]; then + MOCKSERVER_URL="$mock_service_url" python3 - "$mock_expectations" <<'PY' +import json +import os +import sys +import urllib.request + +with open(sys.argv[1], encoding="utf-8") as expectations_file: + expectations = json.load(expectations_file) + +for expectation in expectations: + request = urllib.request.Request( + f'{os.environ["MOCKSERVER_URL"]}/mockserver/expectation', + data=json.dumps(expectation).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="PUT", + ) + with urllib.request.urlopen(request, timeout=10) as response: + if response.status not in (200, 201): + raise RuntimeError(f"MockServer expectation rejected with HTTP {response.status}") +PY +fi + +python3 - "$fixture" "$dynamic_fixture" <<'PY' +import json +import sys + +source, destination = sys.argv[1:] +with open(source, encoding="utf-8") as source_file: + event = json.load(source_file) + +job = event["workflow_job"] +job["id"] = 123457 +job["run_id"] = 654322 +job["run_url"] = job["run_url"].replace("654321", "654322") +job["url"] = job["url"].replace("123456", "123457") +job["html_url"] = job["html_url"].replace("123456", "123457") +job["name"] = "ministack-smoke-dynamic" +job["labels"].append("ghr-ec2-instance-type:m5.large") + +with open(destination, "w", encoding="utf-8") as destination_file: + json.dump(event, destination_file) +PY + +terraform_initialized=true +"$source_root/tests/ministack/run-example.sh" apply default "$tfvars_file" + +printf '%s\n' \ + 'MiniStack smoke chain evidence checklist:' \ + ' [ ] API Gateway accepted the signed workflow_job webhook (HTTP 201)' \ + ' [ ] Webhook Lambda log contains workflow job 123456' \ + ' [ ] EventBridge invoked the dispatcher Lambda (dispatcher log contains 123456)' \ + ' [ ] Dispatcher delivered the job through SQS (scale-up log contains 123456)' \ + ' [ ] Scale-up without a dynamic label called each expected GitHub API route in MockServer' \ + ' [ ] MiniStack EC2 API reports a standard scale-up instance with default EC2 configuration' \ + ' [ ] Scale-up with ghr-ec2-instance-type:m5.large called each expected GitHub API route in MockServer' \ + ' [ ] Dynamic label selected EC2 instance type m5.large' \ + ' [ ] Scale-up EC2 instance has the expected runner discovery tags' \ + ' [ ] Pool called every expected GitHub API route in MockServer' \ + ' [ ] Pool Lambda created a runner instance' \ + ' [ ] Pool EC2 instance has the expected runner discovery tags' \ + ' [ ] Scale-down checks are temporarily disabled pending the MiniStack CreateFleet fix' + +webhook_endpoint=$(terraform -chdir="$example_root" output -raw webhook_endpoint) +endpoint_host_port=${AWS_ENDPOINT_URL#*://} +endpoint_port=${endpoint_host_port##*:} +api_host_port=${webhook_endpoint#*://} +api_host_port=${api_host_port%%/*} +api_host=${api_host_port%:*} +webhook_secret=$(terraform -chdir="$example_root" output -raw webhook_secret) + +send_webhook() { + fixture_file="$1" + delivery_id="$2" + signature=$(openssl dgst -sha256 -hmac "$webhook_secret" "$fixture_file" | awk '{print $NF}') + status_code=$(curl -sS --max-time 15 -o "$response_file" -w '%{http_code}' \ + --connect-to "${api_host}:4566:127.0.0.1:${endpoint_port}" \ + -X POST "$webhook_endpoint" \ + -H 'Content-Type: application/json' \ + -H 'X-GitHub-Event: workflow_job' \ + -H "X-GitHub-Delivery: ${delivery_id}" \ + -H 'X-GitHub-Hook-Installation-Target-ID: 123' \ + -H "X-Hub-Signature-256: sha256=${signature}" \ + --data-binary "@${fixture_file}") + + if [ "$status_code" != 201 ]; then + echo "Webhook smoke request failed with HTTP $status_code." >&2 + sed -n '1,80p' "$response_file" >&2 + exit 1 + fi + echo " [PASS] API Gateway accepted the signed workflow_job webhook ${delivery_id} (HTTP 201)" +} + +send_webhook "$fixture" "ministack-smoke-123456" + +wait_for_log_event() { + log_group="$1" + marker="$2" + description="$3" + attempts=60 + while ! aws --endpoint-url "$AWS_ENDPOINT_URL" logs filter-log-events \ + --log-group-name "$log_group" --limit 50 --output text 2>/dev/null | grep -Fq "$marker"; do + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "Timed out waiting for MiniStack log marker '$marker' in $log_group." >&2 + exit 1 + fi + sleep 2 + done + printf ' [PASS] %s (log group %s contains %s)\n' "$description" "$log_group" "$marker" +} + +wait_for_optional_log_event() { + log_group="$1" + marker="$2" + description="$3" + attempts=60 + while ! aws --endpoint-url "$AWS_ENDPOINT_URL" logs filter-log-events \ + --log-group-name "$log_group" --limit 50 --output text 2>/dev/null | grep -Fq "$marker"; do + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + printf ' [WARN] %s (log marker %s was not observed in %s)\n' \ + "$description" "$marker" "$log_group" + return 0 + fi + sleep 2 + done + printf ' [PASS] %s (log group %s contains %s)\n' "$description" "$log_group" "$marker" +} + +wait_for_log_event "/aws/lambda/ministack-default-webhook" "123456" \ + "Webhook Lambda received workflow job 123456" +wait_for_log_event "/aws/lambda/ministack-default-dispatch-to-runner" "123456" \ + "EventBridge invoked the dispatcher Lambda" +wait_for_log_event "/aws/lambda/ministack-default-scale-up" "123456" \ + "Dispatcher delivered workflow job 123456 through SQS to scale-up" + +wait_for_mock_route() { + method="$1" + route="$2" + description="$3" + verification_body=$(printf '{"httpRequest":{"method":"%s","path":"%s"},"times":{"atLeast":1}}' "$method" "$route") + attempts=60 + while ! curl -fsS --max-time 5 -X PUT "${mock_service_url}/mockserver/verify" \ + -H 'Content-Type: application/json' \ + --data-binary "$verification_body" >/dev/null 2>&1; do + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "Timed out waiting for MockServer route: $method $route" >&2 + curl -sS --max-time 5 -X PUT \ + "${mock_service_url}/mockserver/retrieve?type=REQUEST_RESPONSES&format=JSON" >&2 || true + exit 1 + fi + sleep 2 + done + printf ' [PASS] %s (MockServer verified %s %s)\n' "$description" "$method" "$route" +} + +clear_mock_request_log() { + if ! curl -fsS --max-time 5 -X PUT \ + "${mock_service_url}/mockserver/clear?type=log" >/dev/null 2>&1; then + echo "Failed to clear MockServer request history before the next lifecycle phase." >&2 + exit 1 + fi +} + +assert_scale_down_github_routes() { + wait_for_mock_route POST "/api/v3/app/installations/123/access_tokens" \ + "Scale-down requested a GitHub App installation token" + wait_for_mock_route GET "/api/v3/orgs/test-owner/actions/runners" \ + "Scale-down listed organization runners" + wait_for_mock_route GET "/api/v3/orgs/test-owner/actions/runners/${1}" \ + "Scale-down checked the runner busy state" + wait_for_mock_route DELETE "/api/v3/orgs/test-owner/actions/runners/${1}" \ + "Scale-down deleted the runner from GitHub" +} + +assert_pool_github_routes() { + wait_for_mock_route POST "/api/v3/app/installations/123/access_tokens" \ + "Pool requested a GitHub App installation token" + wait_for_mock_route GET "/api/v3/orgs/test-owner/actions/runners" \ + "Pool listed organization runners" + wait_for_mock_route POST "/api/v3/orgs/test-owner/actions/runners/registration-token" \ + "Pool requested a GitHub runner registration token" +} + +assert_scale_up_github_routes() { + job_id="$1" + wait_for_mock_route POST "/api/v3/app/installations/123/access_tokens" \ + "Scale-up requested a GitHub App installation token for job ${job_id}" + wait_for_mock_route GET "/api/v3/repos/test-owner/test-repo/actions/jobs/${job_id}" \ + "Scale-up checked the queued GitHub job ${job_id}" + wait_for_mock_route POST "/api/v3/orgs/test-owner/actions/runners/registration-token" \ + "Scale-up requested a GitHub runner registration token for job ${job_id}" +} + +assert_scale_up_github_routes 123456 + +wait_for_ec2_instance() { + source="$1" + description="$2" + attempts=60 + while :; do + found_instance_id=$(aws --endpoint-url "$AWS_ENDPOINT_URL" ec2 describe-instances \ + --filters \ + "Name=instance-state-name,Values=running,pending" \ + "Name=tag:ghr:Application,Values=github-action-runner" \ + "Name=tag:ghr:created_by,Values=$source" \ + --query 'Reservations[].Instances[].InstanceId | [0]' \ + --output text 2>/dev/null || true) + if [ -n "$found_instance_id" ] && [ "$found_instance_id" != "None" ]; then + printf ' [PASS] MiniStack EC2 API reports %s: %s\n' "$description" "$found_instance_id" + return + fi + + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "Timed out waiting for $description in the MiniStack EC2 API." >&2 + aws --endpoint-url "$AWS_ENDPOINT_URL" ec2 describe-instances \ + --filters \ + "Name=instance-state-name,Values=running,pending" \ + "Name=tag:ghr:Application,Values=github-action-runner" \ + "Name=tag:ghr:created_by,Values=$source" \ + --output json >&2 || true + exit 1 + fi + sleep 2 + done +} + +wait_for_ec2_instance "scale-up-lambda" "a scale-up instance" +scale_up_instance_id="$found_instance_id" + +assert_ec2_tag() { + instance_id="$1" + key="$2" + expected_value="$3" + description="$4" + actual_value=$(aws --endpoint-url "$AWS_ENDPOINT_URL" ec2 describe-instances \ + --instance-ids "$instance_id" \ + --query "Reservations[].Instances[].Tags[?Key=='${key}'].Value | [0]" \ + --output text 2>/dev/null || true) + if [ "$actual_value" != "$expected_value" ]; then + echo "Expected $description tag $key=$expected_value on $instance_id, got $actual_value." >&2 + exit 1 + fi +} + +assert_ec2_runner_tags() { + instance_id="$1" + source="$2" + description="$3" + assert_ec2_tag "$instance_id" "ghr:Application" "github-action-runner" "$description" + assert_ec2_tag "$instance_id" "ghr:created_by" "$source" "$description" + assert_ec2_tag "$instance_id" "ghr:Type" "Org" "$description" + assert_ec2_tag "$instance_id" "ghr:Owner" "test-owner" "$description" + printf ' [PASS] MiniStack EC2 API reports correct runner tags on %s\n' "$instance_id" +} + +assert_ec2_runner_tags "$scale_up_instance_id" "scale-up-lambda" "the scale-up runner" + +assert_ec2_default_instance_type() { + instance_id="$1" + actual_type=$(aws --endpoint-url "$AWS_ENDPOINT_URL" ec2 describe-instances \ + --instance-ids "$instance_id" \ + --query 'Reservations[0].Instances[0].InstanceType' \ + --output text 2>/dev/null || true) + if [ "$actual_type" != "m7a.large" ]; then + echo "Expected standard scale-up to use the configured default m7a.large, got $actual_type." >&2 + exit 1 + fi + printf ' [PASS] Standard scale-up used the configured default EC2 instance type: %s\n' "$actual_type" +} + +assert_ec2_instance_type() { + instance_id="$1" + expected_type="$2" + actual_type=$(aws --endpoint-url "$AWS_ENDPOINT_URL" ec2 describe-instances \ + --instance-ids "$instance_id" \ + --query 'Reservations[0].Instances[0].InstanceType' \ + --output text 2>/dev/null || true) + if [ "$actual_type" != "$expected_type" ]; then + echo "Expected $instance_id to use EC2 instance type $expected_type, got $actual_type." >&2 + exit 1 + fi + printf ' [PASS] EC2 dynamic label selected instance type %s on %s\n' "$expected_type" "$instance_id" +} + +assert_ec2_default_instance_type "$scale_up_instance_id" + +configure_mock_runner_state() { + instance_id="$1" + runner_id="$2" + MOCKSERVER_URL="$mock_service_url" python3 - "$instance_id" "$runner_id" <<'PY' +import json +import os +import sys +import urllib.request + +instance_id, runner_id = sys.argv[1:] +runner_id = int(runner_id) +base = "/api/v3/orgs/test-owner/actions/runners" + +def control(path, method, payload): + request = urllib.request.Request( + f'{os.environ["MOCKSERVER_URL"]}{path}', + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method=method, + ) + with urllib.request.urlopen(request, timeout=10) as response: + if response.status not in (200, 201, 202): + raise RuntimeError(f'MockServer API rejected {method} {path} with HTTP {response.status}') + +def clear(method, path): + control("/mockserver/clear", "PUT", {"httpRequest": {"method": method, "path": path}}) + +def expect(method, path, status, body=None): + response = {"statusCode": status} + if body is not None: + response["headers"] = {"Content-Type": ["application/json"]} + response["body"] = json.dumps(body) + control( + "/mockserver/expectation", + "PUT", + {"httpRequest": {"method": method, "path": path}, "httpResponse": response}, + ) + +state_path = f"{base}/{runner_id}" +clear("GET", base) +clear("GET", state_path) +clear("DELETE", state_path) +expect( + "GET", + base, + 200, + { + "total_count": 1, + "runners": [ + { + "id": runner_id, + "name": f"ministack-smoke-{instance_id}", + "os": "linux", + "status": "offline", + "busy": False, + "labels": [], + } + ], + }, +) +expect( + "GET", + state_path, + 200, + { + "id": runner_id, + "name": f"ministack-smoke-{instance_id}", + "os": "linux", + "status": "offline", + "busy": False, + "labels": [], + }, +) +expect("DELETE", state_path, 204) +PY +} + +configure_mock_runner_removed() { + runner_id="$1" + MOCKSERVER_URL="$mock_service_url" python3 - "$runner_id" <<'PY' +import json +import os +import sys +import urllib.request + +runner_id = sys.argv[1] +path = f"/api/v3/orgs/test-owner/actions/runners/{runner_id}" + +def control(path, method, payload): + request = urllib.request.Request( + f'{os.environ["MOCKSERVER_URL"]}{path}', + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method=method, + ) + with urllib.request.urlopen(request, timeout=10) as response: + if response.status not in (200, 201, 202): + raise RuntimeError(f'MockServer API rejected {method} {path} with HTTP {response.status}') + +control("/mockserver/clear", "PUT", {"httpRequest": {"method": "GET", "path": path}}) +control( + "/mockserver/expectation", + "PUT", + { + "httpRequest": {"method": "GET", "path": path}, + "httpResponse": { + "statusCode": 404, + "headers": {"Content-Type": ["application/json"]}, + "body": '{"message":"Not Found"}', + }, + }, +) +PY +} + +configure_empty_mock_runner_list() { + MOCKSERVER_URL="$mock_service_url" python3 - <<'PY' +import json +import os +import urllib.request + +path = "/api/v3/orgs/test-owner/actions/runners" + +def control(path, method, payload): + request = urllib.request.Request( + f'{os.environ["MOCKSERVER_URL"]}{path}', + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method=method, + ) + with urllib.request.urlopen(request, timeout=10) as response: + if response.status not in (200, 201, 202): + raise RuntimeError(f'MockServer API rejected {method} {path} with HTTP {response.status}') + +control("/mockserver/clear", "PUT", {"httpRequest": {"method": "GET", "path": path}}) +control( + "/mockserver/expectation", + "PUT", + { + "httpRequest": {"method": "GET", "path": path}, + "httpResponse": { + "statusCode": 200, + "headers": {"Content-Type": ["application/json"]}, + "body": '{"total_count":0,"runners":[]}', + }, + }, +) +PY +} + +assert_mock_runner_removed() { + runner_id="$1" + status_code=$(curl -sS --max-time 5 -o "$response_file" -w '%{http_code}' \ + "${mock_service_url}/api/v3/orgs/test-owner/actions/runners/${runner_id}") + if [ "$status_code" != 404 ]; then + echo "Expected GitHub API mock to return 404 for removed runner $runner_id, got HTTP $status_code." >&2 + sed -n '1,80p' "$response_file" >&2 + exit 1 + fi + printf ' [PASS] GitHub API mock reports runner %s removed (HTTP 404)\n' "$runner_id" +} + +wait_for_ec2_termination() { + instance_id="$1" + description="$2" + attempts=60 + while :; do + state=$(aws --endpoint-url "$AWS_ENDPOINT_URL" ec2 describe-instances \ + --instance-ids "$instance_id" \ + --query 'Reservations[].Instances[].State.Name | [0]' \ + --output text 2>/dev/null || true) + if [ -z "$state" ] || [ "$state" = "None" ] || [ "$state" = "terminated" ]; then + printf ' [PASS] MiniStack EC2 API reports %s terminated\n' "$description" + return + fi + attempts=$((attempts - 1)) + if [ "$attempts" -le 0 ]; then + echo "Timed out waiting for $description to terminate; current state: $state." >&2 + exit 1 + fi + sleep 2 + done +} + +terminate_ec2_instance_directly() { + instance_id="$1" + description="$2" + if ! aws --endpoint-url "$AWS_ENDPOINT_URL" ec2 terminate-instances \ + --instance-ids "$instance_id" >/dev/null; then + echo "Failed to terminate $description through the MiniStack EC2 API." >&2 + exit 1 + fi + wait_for_ec2_termination "$instance_id" "$description" +} + +invoke_lambda() { + function_name="$1" + payload="$2" + description="$3" + invocation_result=$(aws --endpoint-url "$AWS_ENDPOINT_URL" lambda invoke \ + --cli-binary-format raw-in-base64-out \ + --invocation-type RequestResponse \ + --function-name "$function_name" \ + --payload "$payload" \ + "$lambda_response_file" --output json) + if printf '%s' "$invocation_result" | grep -Fq '"FunctionError"'; then + echo "Lambda invocation returned FunctionError for $function_name." >&2 + exit 1 + fi + printf ' [PASS] %s (Lambda API accepted the request)\n' "$description" +} + +# Temporarily disabled until MiniStack fixes its CreateFleet behavior. Keep the +# lifecycle assertions here so this coverage can be restored with the fix. +# scale_up_runner_id=987654321 +# configure_mock_runner_state "$scale_up_instance_id" "$scale_up_runner_id" +# clear_mock_request_log +# invoke_lambda "ministack-default-scale-down" '{"smokeMarker":"ministack-scale-up-scale-down"}' \ +# "Scale-down Lambda invoked for the scale-up runner" +# wait_for_log_event "/aws/lambda/ministack-default-scale-down" "ministack-scale-up-scale-down" \ +# "Scale-down Lambda started processing the scale-up runner" +# assert_scale_down_github_routes "$scale_up_runner_id" +# configure_mock_runner_removed "$scale_up_runner_id" +# assert_mock_runner_removed "$scale_up_runner_id" +# wait_for_ec2_termination "$scale_up_instance_id" "the scale-up instance" +# wait_for_optional_log_event "/aws/lambda/ministack-default-scale-down" "$scale_up_instance_id" \ +# "Scale-down log recorded termination of the scale-up EC2 runner" + +# Temporary cleanup workaround for MiniStack CreateFleet issue #1678. Directly +# terminate the test-created instance so the next scale-up is not blocked by +# runners_maximum_count or mistaken for this instance. +terminate_ec2_instance_directly "$scale_up_instance_id" "the standard scale-up instance" + +clear_mock_request_log +send_webhook "$dynamic_fixture" "ministack-smoke-123457" +wait_for_log_event "/aws/lambda/ministack-default-webhook" "123457" \ + "Webhook Lambda received dynamic-label workflow job 123457" +wait_for_log_event "/aws/lambda/ministack-default-dispatch-to-runner" "123457" \ + "EventBridge invoked the dispatcher for dynamic-label workflow job 123457" +wait_for_log_event "/aws/lambda/ministack-default-scale-up" "123457" \ + "Dispatcher delivered dynamic-label workflow job 123457 through SQS to scale-up" +assert_scale_up_github_routes 123457 +wait_for_ec2_instance "scale-up-lambda" "a dynamic-label scale-up instance" +dynamic_scale_up_instance_id="$found_instance_id" +assert_ec2_runner_tags "$dynamic_scale_up_instance_id" "scale-up-lambda" \ + "the dynamic-label scale-up runner" +assert_ec2_instance_type "$dynamic_scale_up_instance_id" "m5.large" + +# Temporarily disabled for the same MiniStack CreateFleet issue. +# dynamic_scale_up_runner_id=987654323 +# configure_mock_runner_state "$dynamic_scale_up_instance_id" "$dynamic_scale_up_runner_id" +# clear_mock_request_log +# invoke_lambda "ministack-default-scale-down" '{"smokeMarker":"ministack-dynamic-scale-up-scale-down"}' \ +# "Scale-down Lambda invoked for the dynamic-label scale-up runner" +# wait_for_log_event "/aws/lambda/ministack-default-scale-down" "ministack-dynamic-scale-up-scale-down" \ +# "Scale-down Lambda started processing the dynamic-label scale-up runner" +# assert_scale_down_github_routes "$dynamic_scale_up_runner_id" +# configure_mock_runner_removed "$dynamic_scale_up_runner_id" +# assert_mock_runner_removed "$dynamic_scale_up_runner_id" +# wait_for_ec2_termination "$dynamic_scale_up_instance_id" "the dynamic-label scale-up instance" +# wait_for_optional_log_event "/aws/lambda/ministack-default-scale-down" "$dynamic_scale_up_instance_id" \ +# "Scale-down log recorded termination of the dynamic-label scale-up EC2 runner" + +terminate_ec2_instance_directly "$dynamic_scale_up_instance_id" "the dynamic-label scale-up instance" + +echo "MiniStack smoke chain 1 passed: API Gateway -> webhook -> EventBridge -> dispatcher -> SQS -> scale-up without and with EC2 dynamic label -> GitHub API mock." + +configure_empty_mock_runner_list +clear_mock_request_log +invoke_lambda "ministack-default-pool" '{"poolSize":1,"type":"ec2"}' \ + "Pool Lambda invoked to maintain one runner" +assert_pool_github_routes +wait_for_log_event "/aws/lambda/ministack-default-pool" "topped up with 1 runners" \ + "Pool Lambda requested one runner" +wait_for_ec2_instance "pool-lambda" "a pool instance" +pool_instance_id="$found_instance_id" +assert_ec2_runner_tags "$pool_instance_id" "pool-lambda" "the pool runner" + +# Temporarily disabled for the same MiniStack CreateFleet issue. +# pool_runner_id=987654322 +# configure_mock_runner_state "$pool_instance_id" "$pool_runner_id" +# clear_mock_request_log +# invoke_lambda "ministack-default-scale-down" '{"smokeMarker":"ministack-pool-scale-down"}' \ +# "Scale-down Lambda invoked for the pool runner" +# wait_for_log_event "/aws/lambda/ministack-default-scale-down" "ministack-pool-scale-down" \ +# "Scale-down Lambda started processing the pool runner" +# assert_scale_down_github_routes "$pool_runner_id" +# configure_mock_runner_removed "$pool_runner_id" +# assert_mock_runner_removed "$pool_runner_id" +# wait_for_ec2_termination "$pool_instance_id" "the pool instance" +# wait_for_optional_log_event "/aws/lambda/ministack-default-scale-down" "$pool_instance_id" \ +# "Scale-down log recorded termination of the pool EC2 runner" + +terminate_ec2_instance_directly "$pool_instance_id" "the pool instance" + +echo "MiniStack smoke chain 2 passed: pool -> GitHub API mock -> EC2 runner creation." +echo "MiniStack smoke tests passed: scale-up and pool lifecycle checks completed; scale-down checks are temporarily disabled." diff --git a/tests/ministack/workflow_job_event.json b/tests/ministack/workflow_job_event.json new file mode 100644 index 0000000000..6200b40fb1 --- /dev/null +++ b/tests/ministack/workflow_job_event.json @@ -0,0 +1,26 @@ +{ + "action": "queued", + "workflow_job": { + "id": 123456, + "run_id": 654321, + "run_url": "https://github.example.invalid/test-owner/test-repo/actions/runs/654321", + "url": "https://github.example.invalid/test-owner/test-repo/actions/jobs/123456", + "html_url": "https://github.example.invalid/test-owner/test-repo/actions/jobs/123456", + "status": "queued", + "conclusion": null, + "name": "ministack-smoke", + "labels": ["self-hosted", "linux", "x64", "default", "example"] + }, + "repository": { + "id": 1, + "name": "test-repo", + "full_name": "test-owner/test-repo", + "owner": { + "login": "test-owner", + "type": "Organization" + } + }, + "installation": { + "id": 123 + } +}