From 0f8059bbd45f982e5ba2d7194d7d9c13e57135fb Mon Sep 17 00:00:00 2001 From: Sirajmx Date: Wed, 22 Jul 2026 23:38:36 +0530 Subject: [PATCH 1/4] =?UTF-8?q?infra:=20wire=20ECS=20staging=20deploy=20?= =?UTF-8?q?=E2=80=94=20fix=20naming,=20add=20OIDC=20role,=20fix=20Dockerfi?= =?UTF-8?q?le?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deploy.yml: - Fix ECR_REPOSITORY, ECS_CLUSTER, ECS_SERVICE to match Terraform resource names (add missing 'system-' prefix and env-scoped ECR path) - Pass GH_PAT secret to docker build for private iab-agentic-primitives dep Dockerfile: - Add git to builder apt deps (needed for VCS pip install) - Use BuildKit --mount=type=secret for GitHub token; token never written to any image layer (GIT_CONFIG_COUNT env-var approach, no gitconfig file) - Runtime stage: --no-deps on pip install avoids re-cloning git deps (all deps already resolved in the copied venv from builder) Terraform: - Add github_oidc.tf: OIDC provider + scoped IAM deploy role for GitHub Actions (ECR push + ECS update-service); after apply, set output github_deploy_role_arn as AWS_DEPLOY_ROLE_ARN in repo secrets - Expose ecr_repository_arn and ecs_cluster_arn from compute module so the OIDC policy can reference them without hardcoding ARN strings --- .github/workflows/deploy.yml | 8 +- infra/aws/terraform/github_oidc.tf | 127 ++++++++++++++++++ .../aws/terraform/modules/compute/outputs.tf | 10 ++ infra/aws/terraform/outputs.tf | 5 + infra/docker/Dockerfile | 37 +++-- 5 files changed, 173 insertions(+), 14 deletions(-) create mode 100644 infra/aws/terraform/github_oidc.tf diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 57a1ed16..4fdd7701 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -24,9 +24,9 @@ permissions: env: AWS_REGION: us-east-1 - ECR_REPOSITORY: ad-buyer-system - ECS_CLUSTER: ad-buyer-${{ inputs.environment || 'staging' }}-cluster - ECS_SERVICE: ad-buyer-${{ inputs.environment || 'staging' }}-service + ECR_REPOSITORY: ad-buyer-system/${{ inputs.environment || 'staging' }} + ECS_CLUSTER: ad-buyer-system-${{ inputs.environment || 'staging' }}-cluster + ECS_SERVICE: ad-buyer-system-${{ inputs.environment || 'staging' }}-service jobs: deploy: @@ -52,8 +52,10 @@ jobs: env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} IMAGE_TAG: ${{ github.sha }} + GITHUB_TOKEN: ${{ secrets.GH_PAT }} run: | docker build \ + --secret id=github_token,env=GITHUB_TOKEN \ -f infra/docker/Dockerfile \ -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG \ -t $ECR_REGISTRY/$ECR_REPOSITORY:latest \ diff --git a/infra/aws/terraform/github_oidc.tf b/infra/aws/terraform/github_oidc.tf new file mode 100644 index 00000000..01e61ef9 --- /dev/null +++ b/infra/aws/terraform/github_oidc.tf @@ -0,0 +1,127 @@ +# Author: Green Mountain Systems AI Inc. +# Donated to IAB Tech Lab + +# GitHub Actions OIDC provider + deploy role. +# +# This allows GitHub Actions to assume an AWS IAM role via OIDC (no static +# long-lived credentials stored in GitHub secrets). After `terraform apply`, +# copy the `github_deploy_role_arn` output into the repo's GitHub secret: +# Settings → Secrets → Actions → AWS_DEPLOY_ROLE_ARN + +variable "github_repo" { + description = "GitHub repository in owner/repo format (e.g. IABTechLab/buyer-agent)" + type = string + default = "IABTechLab/buyer-agent" +} + +data "aws_iam_openid_connect_provider" "github" { + count = 1 + url = "https://token.actions.githubusercontent.com" +} + +# Create the OIDC provider only when it doesn't already exist in the account. +resource "aws_iam_openid_connect_provider" "github" { + count = length(data.aws_iam_openid_connect_provider.github) == 0 ? 1 : 0 + + url = "https://token.actions.githubusercontent.com" + client_id_list = ["sts.amazonaws.com"] + thumbprint_list = ["6938fd4d98bab03faadb97b34396831e3780aea1"] +} + +locals { + oidc_provider_arn = ( + length(data.aws_iam_openid_connect_provider.github) > 0 + ? data.aws_iam_openid_connect_provider.github[0].arn + : aws_iam_openid_connect_provider.github[0].arn + ) +} + +data "aws_iam_policy_document" "github_deploy_assume" { + statement { + effect = "Allow" + actions = ["sts:AssumeRoleWithWebIdentity"] + + principals { + type = "Federated" + identifiers = [local.oidc_provider_arn] + } + + condition { + test = "StringEquals" + variable = "token.actions.githubusercontent.com:aud" + values = ["sts.amazonaws.com"] + } + + # Allow the deploy job on main or any workflow_dispatch from the repo. + condition { + test = "StringLike" + variable = "token.actions.githubusercontent.com:sub" + values = ["repo:${var.github_repo}:*"] + } + } +} + +resource "aws_iam_role" "github_deploy" { + name = "ad-buyer-system-github-deploy" + assume_role_policy = data.aws_iam_policy_document.github_deploy_assume.json + description = "Assumed by GitHub Actions via OIDC to deploy buyer-agent to ECS" + + tags = { + Project = "ad-buyer-system" + ManagedBy = "terraform" + } +} + +data "aws_iam_policy_document" "github_deploy_permissions" { + # ECR — push images + statement { + effect = "Allow" + actions = [ + "ecr:GetAuthorizationToken", + ] + resources = ["*"] + } + + statement { + effect = "Allow" + actions = [ + "ecr:BatchCheckLayerAvailability", + "ecr:InitiateLayerUpload", + "ecr:UploadLayerPart", + "ecr:CompleteLayerUpload", + "ecr:PutImage", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer", + ] + resources = [module.compute.ecr_repository_arn] + } + + # ECS — trigger rolling deploy + wait for stability + statement { + effect = "Allow" + actions = [ + "ecs:UpdateService", + "ecs:DescribeServices", + "ecs:DescribeTasks", + "ecs:DescribeTaskDefinition", + "ecs:ListTasks", + ] + resources = ["*"] + condition { + test = "ArnLike" + variable = "ecs:cluster" + values = [module.compute.ecs_cluster_arn] + } + } +} + +resource "aws_iam_role_policy" "github_deploy" { + name = "ad-buyer-system-github-deploy-policy" + role = aws_iam_role.github_deploy.id + policy = data.aws_iam_policy_document.github_deploy_permissions.json +} + +output "github_deploy_role_arn" { + description = "ARN to set as AWS_DEPLOY_ROLE_ARN in GitHub Actions secrets" + value = aws_iam_role.github_deploy.arn +} diff --git a/infra/aws/terraform/modules/compute/outputs.tf b/infra/aws/terraform/modules/compute/outputs.tf index 532a8fd1..e9dbd5d5 100644 --- a/infra/aws/terraform/modules/compute/outputs.tf +++ b/infra/aws/terraform/modules/compute/outputs.tf @@ -13,11 +13,21 @@ output "ecr_repository_url" { value = aws_ecr_repository.this.repository_url } +output "ecr_repository_arn" { + description = "ARN of the ECR repository (used by the GitHub deploy IAM role)" + value = aws_ecr_repository.this.arn +} + output "ecs_cluster_name" { description = "Name of the ECS cluster" value = aws_ecs_cluster.this.name } +output "ecs_cluster_arn" { + description = "ARN of the ECS cluster (used by the GitHub deploy IAM role)" + value = aws_ecs_cluster.this.arn +} + output "ecs_service_name" { description = "Name of the ECS service" value = aws_ecs_service.this.name diff --git a/infra/aws/terraform/outputs.tf b/infra/aws/terraform/outputs.tf index 960db0dd..6646b10f 100644 --- a/infra/aws/terraform/outputs.tf +++ b/infra/aws/terraform/outputs.tf @@ -23,6 +23,11 @@ output "ecs_cluster_name" { value = module.compute.ecs_cluster_name } +output "ecs_cluster_arn" { + description = "ARN of the ECS cluster" + value = module.compute.ecs_cluster_arn +} + output "ecs_service_name" { description = "Name of the ECS service" value = module.compute.ecs_service_name diff --git a/infra/docker/Dockerfile b/infra/docker/Dockerfile index 22a0f993..979c8cea 100644 --- a/infra/docker/Dockerfile +++ b/infra/docker/Dockerfile @@ -3,6 +3,15 @@ # ============================================================================= # Stage 1: Build dependencies (cached layer) # Stage 2: Production runtime (minimal image) +# +# Private git deps (iab-agentic-primitives) require a GitHub token. +# Pass it at build time via BuildKit secret — never baked into any layer: +# +# docker build \ +# --secret id=github_token,env=GITHUB_TOKEN \ +# -f infra/docker/Dockerfile -t buyer-agent:local . +# +# In GitHub Actions the GITHUB_TOKEN is passed automatically (see deploy.yml). # ============================================================================= # --------------------------------------------------------------------------- @@ -14,21 +23,27 @@ WORKDIR /build RUN apt-get update && apt-get install -y --no-install-recommends \ gcc \ + git \ && rm -rf /var/lib/apt/lists/* -# Copy only dependency metadata first (cache-friendly) -COPY pyproject.toml ./ - # Install dependencies into a virtual env so we can copy it cleanly RUN python -m venv /opt/venv ENV PATH="/opt/venv/bin:$PATH" -RUN pip install --no-cache-dir --upgrade pip \ - && pip install --no-cache-dir -e . 2>/dev/null; true +RUN pip install --no-cache-dir --upgrade pip + +# Copy source and install all deps. +# The GitHub token is mounted as a BuildKit secret — used only during this +# RUN step and never written to any image layer. +COPY pyproject.toml README.md ./ +COPY src ./src -# Now copy source and install the package itself -COPY . . -RUN pip install --no-cache-dir -e . +RUN --mount=type=secret,id=github_token \ + export GH_TOKEN=$(cat /run/secrets/github_token) && \ + GIT_CONFIG_COUNT=1 \ + GIT_CONFIG_KEY_0="url.https://${GH_TOKEN}@github.com/.insteadOf" \ + GIT_CONFIG_VALUE_0="https://github.com/" \ + pip install --no-cache-dir -e . # --------------------------------------------------------------------------- # Stage 2 — Runtime @@ -43,7 +58,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ RUN groupadd --gid 1000 buyer && \ useradd --uid 1000 --gid buyer --create-home buyer -# Copy virtual env from builder +# Copy virtual env from builder (all deps already resolved, no re-clone needed) COPY --from=builder /opt/venv /opt/venv ENV PATH="/opt/venv/bin:$PATH" @@ -53,8 +68,8 @@ COPY --from=builder /build/src ./src COPY --from=builder /build/pyproject.toml ./ COPY --from=builder /build/README.md ./ -# Install the package in the runtime venv -RUN pip install --no-cache-dir -e . +# Register the editable package — deps already in venv, skip resolution +RUN pip install --no-cache-dir --no-deps -e . # Create data directory for SQLite (owned by non-root user) RUN mkdir -p /app/data && chown buyer:buyer /app/data From 98bd1aa16d1e2ac72974a67dcabe91ab678f22a0 Mon Sep 17 00:00:00 2001 From: Sirajmx Date: Thu, 23 Jul 2026 00:18:30 +0530 Subject: [PATCH 2/4] infra: replace dynamodb lock with s3 use_lockfile; trigger deploy on branch --- .github/workflows/deploy.yml | 2 +- infra/aws/terraform/main.tf | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 4fdd7701..6798bfac 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -2,7 +2,7 @@ name: Deploy on: push: - branches: [main] + branches: [main, infra/wire-ecs-staging] paths-ignore: - 'docs/**' - '*.md' diff --git a/infra/aws/terraform/main.tf b/infra/aws/terraform/main.tf index 45280faa..61bb626b 100644 --- a/infra/aws/terraform/main.tf +++ b/infra/aws/terraform/main.tf @@ -9,11 +9,11 @@ terraform { } backend "s3" { - bucket = "ad-buyer-system-terraform-state" - key = "terraform.tfstate" - region = "us-east-1" - dynamodb_table = "ad-buyer-system-terraform-lock" - encrypt = true + bucket = "ad-buyer-system-terraform-state" + key = "terraform.tfstate" + region = "us-east-1" + encrypt = true + use_lockfile = true } } From 01274b8cf4e5314ef14702cfc39e47960f10c99d Mon Sep 17 00:00:00 2001 From: Sirajmx Date: Thu, 23 Jul 2026 00:23:04 +0530 Subject: [PATCH 3/4] fix: remove data source for OIDC provider; create it directly --- infra/aws/terraform/github_oidc.tf | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/infra/aws/terraform/github_oidc.tf b/infra/aws/terraform/github_oidc.tf index 01e61ef9..57ed0c7f 100644 --- a/infra/aws/terraform/github_oidc.tf +++ b/infra/aws/terraform/github_oidc.tf @@ -14,26 +14,14 @@ variable "github_repo" { default = "IABTechLab/buyer-agent" } -data "aws_iam_openid_connect_provider" "github" { - count = 1 - url = "https://token.actions.githubusercontent.com" -} - -# Create the OIDC provider only when it doesn't already exist in the account. resource "aws_iam_openid_connect_provider" "github" { - count = length(data.aws_iam_openid_connect_provider.github) == 0 ? 1 : 0 - url = "https://token.actions.githubusercontent.com" client_id_list = ["sts.amazonaws.com"] thumbprint_list = ["6938fd4d98bab03faadb97b34396831e3780aea1"] } locals { - oidc_provider_arn = ( - length(data.aws_iam_openid_connect_provider.github) > 0 - ? data.aws_iam_openid_connect_provider.github[0].arn - : aws_iam_openid_connect_provider.github[0].arn - ) + oidc_provider_arn = aws_iam_openid_connect_provider.github.arn } data "aws_iam_policy_document" "github_deploy_assume" { From 8381afbf3e858ae307bc4af77c2777d0f64e40be Mon Sep 17 00:00:00 2001 From: Sirajmx Date: Thu, 23 Jul 2026 01:58:02 +0530 Subject: [PATCH 4/4] restrict deployment to main --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 6798bfac..4fdd7701 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -2,7 +2,7 @@ name: Deploy on: push: - branches: [main, infra/wire-ecs-staging] + branches: [main] paths-ignore: - 'docs/**' - '*.md'