From 145be75a088b3855ffa07474ac4fe50164ecd1f7 Mon Sep 17 00:00:00 2001 From: Cristian Magherusan-Stanciu Date: Mon, 28 Sep 2026 01:41:12 +0200 Subject: [PATCH 1/2] fix(ci): run database-migration.yml terraform init with -backend-config All three environments declare a partial backend (bucket/prefix/key provided via -backend-config, per terraform/environments/*/backend.tf), but database-migration.yml ran bare `terraform init` with no -backend-config and never referenced secrets.TF_BACKEND_*. Terraform cannot prompt for the missing backend attributes on a non-interactive runner, so init failed before a single migration could run -- the documented safe migration path was unusable, leaving only the concurrent-at-deploy-time DB_AUTO_MIGRATE path that produces the "dirty" golang-migrate state this repo keeps a manual recovery runbook for. Fix, mirroring deploy-aws-lambda.yml/deploy-gcp.yml/deploy-azure.yml: - Add a pinned `hashicorp/setup-terraform` step to all three jobs (nothing previously put terraform on PATH at all; whether it existed depended on whatever ubuntu-latest happened to ship). - Write the per-cloud .tfbackend file from secrets.TF_BACKEND_{AWS,GCP, AZURE} and pass -backend-config, using the same key/prefix format the matching deploy workflow uses for that cloud. - Drop the `2>/dev/null || echo ""` around `terraform output -raw`. Once init succeeds, that pattern turns every distinct cause (output renamed, empty state for the environment, wrong key, insufficient IAM) into the same undifferentiated "Failed to get database endpoint" -- a bad property for the step right before a schema migration. AWS-specific: terraform/environments/aws is one root applied into two independent state objects (github-/ for Lambda, github-fargate- / for Fargate), each with its own RDS instance since module.database carries no `count` on compute_platform. Picking the wrong one silently migrates a database nobody is serving traffic from while the live one stays on the old schema. Rather than hardcoding the Lambda key, add an explicit `compute_platform` input (default lambda, matching this repo's primary compute platform), re-validated in the `validate` job's safety checks the same way cloud/environment/direction already are, and use it to select the state key. Verified: actionlint (native v1.7.12, exit 0) and zizmor (--offline --persona=pedantic --min-severity=medium, 0 findings) on database-migration.yml. Closes #114 Co-Authored-By: claude-flow --- .github/workflows/database-migration.yml | 105 +++++++++++++++++++++-- 1 file changed, 96 insertions(+), 9 deletions(-) diff --git a/.github/workflows/database-migration.yml b/.github/workflows/database-migration.yml index 973f68772..b5e7d64ef 100644 --- a/.github/workflows/database-migration.yml +++ b/.github/workflows/database-migration.yml @@ -60,6 +60,12 @@ on: required: false type: string default: '' + compute_platform: + description: 'AWS compute platform whose database to migrate (terraform/environments/aws has two independent RDS instances, one per namespace; ignored for gcp/azure)' + required: false + type: choice + options: [lambda, fargate] + default: lambda workflow_call: inputs: cloud: @@ -72,9 +78,16 @@ on: required: false type: string default: up + compute_platform: + required: false + type: string + default: lambda env: MIGRATIONS_PATH: internal/database/postgres/migrations + # Must satisfy `required_version = ">= 1.10.0"` declared by every + # terraform/environments/*/main.tf. Keep in sync with ci.yml/pre-commit.yml. + TF_VERSION: '1.10.5' jobs: # Validate migration request @@ -100,6 +113,7 @@ jobs: DIRECTION: ${{ inputs.direction }} STEPS: ${{ inputs.steps }} CONFIRM: ${{ inputs.confirm }} + COMPUTE_PLATFORM: ${{ inputs.compute_platform }} run: | set -uo pipefail IS_SAFE=true @@ -109,6 +123,11 @@ jobs: # (apply/rollback everything) rather than rejecting it outright. STEPS="${STEPS:-0}" + # Both triggers default `compute_platform` to "lambda", but + # workflow_call's is a plain `type: string` with no `choice` + # enforcement, so re-validate it here like every other input. + COMPUTE_PLATFORM="${COMPUTE_PLATFORM:-lambda}" + # workflow_dispatch's cloud/environment/direction are `type: choice`, # enforced server-side before the run starts. workflow_call's are plain # `type: string` with no such enforcement, so re-validate them here @@ -137,6 +156,14 @@ jobs: ;; esac + case "$COMPUTE_PLATFORM" in + lambda|fargate) ;; + *) + echo "❌ Invalid compute_platform: '$COMPUTE_PLATFORM' (expected lambda or fargate)" + IS_SAFE=false + ;; + esac + # Check if migrations directory exists if [ ! -d "$MIGRATIONS_PATH" ]; then echo "❌ Migrations directory not found: $MIGRATIONS_PATH" @@ -289,14 +316,48 @@ jobs: role-to-assume: ${{ vars.AWS_ROLE_TO_ASSUME }} aws-region: ${{ vars.AWS_REGION || 'us-east-1' }} + - name: Setup Terraform + uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 + with: + terraform_version: ${{ env.TF_VERSION }} + - name: Get database endpoint from Terraform id: get-endpoint + env: + TF_BACKEND: ${{ secrets.TF_BACKEND_AWS }} + ENVIRONMENT: ${{ inputs.environment }} + COMPUTE_PLATFORM: ${{ inputs.compute_platform || 'lambda' }} run: | + set -euo pipefail cd terraform/environments/aws - terraform init - DB_ENDPOINT=$(terraform output -raw database_proxy_endpoint 2>/dev/null || echo "") + + # terraform/environments/aws is one root applied into two independent + # state objects, each with its own RDS instance (module.database has + # no `count` on compute_platform): github-/ for Lambda deploys, + # github-fargate-/ for Fargate. Picking the wrong one migrates a + # database nobody is serving traffic from while the live one stays on + # the old schema, so the namespace is an explicit operator input + # (`inputs.compute_platform`), never guessed. `validate` already + # rejects anything outside lambda|fargate before this job runs. + case "$COMPUTE_PLATFORM" in + lambda) STATE_KEY="github-${ENVIRONMENT}/terraform.tfstate" ;; + fargate) STATE_KEY="github-fargate-${ENVIRONMENT}/terraform.tfstate" ;; + *) + echo "::error::Unknown compute_platform: '$COMPUTE_PLATFORM'" + exit 1 + ;; + esac + printf '%s\nkey = "%s"\n' "$TF_BACKEND" "$STATE_KEY" > /tmp/backend.tfbackend + terraform init -backend-config=/tmp/backend.tfbackend + + # No `2>/dev/null || echo ""` here: letting terraform's own error + # reach the log is what tells an operator WHICH cause they hit + # (output renamed, empty state for this environment/platform, wrong + # key, insufficient IAM on the state bucket) instead of the single + # undifferentiated "Failed to get database endpoint" below. + DB_ENDPOINT=$(terraform output -raw database_proxy_endpoint) if [ -z "$DB_ENDPOINT" ]; then - echo "Failed to get database endpoint" + echo "::error::terraform output returned an empty database_proxy_endpoint for compute_platform=$COMPUTE_PLATFORM, environment=$ENVIRONMENT" exit 1 fi echo "endpoint=$DB_ENDPOINT" >> "$GITHUB_OUTPUT" @@ -403,14 +464,27 @@ jobs: - name: Set up Cloud SDK uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 + - name: Setup Terraform + uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 + with: + terraform_version: ${{ env.TF_VERSION }} + - name: Get database endpoint from Terraform id: get-endpoint + env: + TF_BACKEND: ${{ secrets.TF_BACKEND_GCP }} + ENVIRONMENT: ${{ inputs.environment }} run: | + set -euo pipefail + printf '%s\nprefix = "github-%s"\n' "$TF_BACKEND" "$ENVIRONMENT" > /tmp/backend.tfbackend cd terraform/environments/gcp - terraform init - DB_ENDPOINT=$(terraform output -raw database_private_ip 2>/dev/null || echo "") + terraform init -backend-config=/tmp/backend.tfbackend + + # No `2>/dev/null || echo ""`: let terraform's own error reach the + # log instead of collapsing every distinct cause into one message. + DB_ENDPOINT=$(terraform output -raw database_private_ip) if [ -z "$DB_ENDPOINT" ]; then - echo "Failed to get database endpoint" + echo "::error::terraform output returned an empty database_private_ip for environment=$ENVIRONMENT" exit 1 fi echo "endpoint=$DB_ENDPOINT" >> "$GITHUB_OUTPUT" @@ -495,14 +569,27 @@ jobs: tenant-id: ${{ secrets.AZURE_TENANT_ID }} subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + - name: Setup Terraform + uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 + with: + terraform_version: ${{ env.TF_VERSION }} + - name: Get database endpoint from Terraform id: get-endpoint + env: + TF_BACKEND: ${{ secrets.TF_BACKEND_AZURE }} + ENVIRONMENT: ${{ inputs.environment }} run: | + set -euo pipefail + printf '%s\nkey = "github-%s.terraform.tfstate"\n' "$TF_BACKEND" "$ENVIRONMENT" > /tmp/backend.tfbackend cd terraform/environments/azure - terraform init - DB_ENDPOINT=$(terraform output -raw database_fqdn 2>/dev/null || echo "") + terraform init -backend-config=/tmp/backend.tfbackend + + # No `2>/dev/null || echo ""`: let terraform's own error reach the + # log instead of collapsing every distinct cause into one message. + DB_ENDPOINT=$(terraform output -raw database_fqdn) if [ -z "$DB_ENDPOINT" ]; then - echo "Failed to get database endpoint" + echo "::error::terraform output returned an empty database_fqdn for environment=$ENVIRONMENT" exit 1 fi echo "endpoint=$DB_ENDPOINT" >> "$GITHUB_OUTPUT" From a53d3837981e112025bec25f2b06195d4be18afe Mon Sep 17 00:00:00 2001 From: Cristian Magherusan-Stanciu Date: Mon, 28 Sep 2026 04:04:14 +0200 Subject: [PATCH 2/2] fix(ci): use the real database_server_fqdn output for Azure migrations Independent review caught that database_fqdn is not a root Terraform output in terraform/environments/azure -- it's only a key inside the deployment_summary map (outputs.tf:193). The only root output that exposes the server hostname is database_server_fqdn (outputs.tf:82). terraform output -raw database_fqdn would fail with "no such output" regardless of the -backend-config fix, so migrate-azure would still never reach `migrate`. Verified AWS (database_proxy_endpoint, outputs.tf:64) and GCP (database_private_ip, outputs.tf:34) already reference real root outputs -- only Azure was wrong. Co-Authored-By: claude-flow --- .github/workflows/database-migration.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/database-migration.yml b/.github/workflows/database-migration.yml index b5e7d64ef..6cd5fceba 100644 --- a/.github/workflows/database-migration.yml +++ b/.github/workflows/database-migration.yml @@ -587,9 +587,9 @@ jobs: # No `2>/dev/null || echo ""`: let terraform's own error reach the # log instead of collapsing every distinct cause into one message. - DB_ENDPOINT=$(terraform output -raw database_fqdn) + DB_ENDPOINT=$(terraform output -raw database_server_fqdn) if [ -z "$DB_ENDPOINT" ]; then - echo "::error::terraform output returned an empty database_fqdn for environment=$ENVIRONMENT" + echo "::error::terraform output returned an empty database_server_fqdn for environment=$ENVIRONMENT" exit 1 fi echo "endpoint=$DB_ENDPOINT" >> "$GITHUB_OUTPUT"