diff --git a/.github/workflows/demo-proof.yml b/.github/workflows/demo-proof.yml index 53eb4df..197e0b3 100644 --- a/.github/workflows/demo-proof.yml +++ b/.github/workflows/demo-proof.yml @@ -24,6 +24,7 @@ jobs: MODELFORGE_URL: ${{ vars.MODELFORGE_PUBLIC_URL }} MODELFORGE_API_KEY: ${{ secrets.MODELFORGE_DEMO_API_KEY }} MODELFORGE_WORKSPACE: ${{ vars.MODELFORGE_DEMO_WORKSPACE }} + MODELFORGE_PROOF_ENVIRONMENT: ${{ inputs.environment }} steps: - uses: actions/checkout@v4 @@ -52,6 +53,11 @@ jobs: exit 1 fi + if [[ ! "${MODELFORGE_PROOF_ENVIRONMENT}" =~ ^[a-zA-Z0-9][a-zA-Z0-9_-]{0,47}$ ]]; then + echo "Proof environment must be 1-48 letters, digits, hyphens, or underscores and start with a letter or digit." >&2 + exit 1 + fi + - name: Install proof client run: python -m pip install -e . @@ -67,7 +73,12 @@ jobs: shell: bash run: | set -euo pipefail - python demo/proof_scenario.py --environment "${{ inputs.environment }}-${GITHUB_RUN_ID}" --weight 50 --traffic-requests 60 --output-dir hosted-demo-proof 2>&1 | tee hosted-demo-proof.log + python demo/proof_scenario.py \ + --environment "${MODELFORGE_PROOF_ENVIRONMENT}-${GITHUB_RUN_ID}" \ + --weight 50 \ + --traffic-requests 60 \ + --output-dir hosted-demo-proof \ + 2>&1 | tee hosted-demo-proof.log - name: Upload hosted proof evidence if: always() diff --git a/.github/workflows/release-aws.yml b/.github/workflows/release-aws.yml index 5edfb0b..1fe174d 100644 --- a/.github/workflows/release-aws.yml +++ b/.github/workflows/release-aws.yml @@ -128,7 +128,11 @@ jobs: if aws ecr describe-images --repository-name "${{ steps.release.outputs.api_repository }}" --image-ids "imageTag=${{ steps.release.outputs.tag }}" >/dev/null 2>&1; then echo "API image already exists for this commit; reusing immutable tag." else - docker build --platform linux/amd64 --tag "${{ steps.release.outputs.api_image }}" . + docker build \ + --platform linux/amd64 \ + --build-arg "MODELFORGE_BUILD_SHA=${GITHUB_SHA}" \ + --tag "${{ steps.release.outputs.api_image }}" \ + . docker push "${{ steps.release.outputs.api_image }}" fi @@ -279,14 +283,22 @@ jobs: BASE_URL="${MODELFORGE_PUBLIC_URL%/}" for attempt in $(seq 1 24); do - if curl --fail --silent --show-error "${BASE_URL}/health" >/dev/null && curl --fail --silent --show-error "${BASE_URL}/ready" >/dev/null; then - printf '{"status":"healthy","url":"%s","image_tag":"%s","commit":"%s"}\n' "${BASE_URL}" "${{ steps.release.outputs.tag }}" "${GITHUB_SHA}" > release-summary.json + if HEALTH_JSON="$(curl --fail --silent --show-error "${BASE_URL}/health")" \ + && jq -e --arg sha "${GITHUB_SHA}" \ + '.status == "healthy" and .build_sha == $sha' \ + <<<"${HEALTH_JSON}" >/dev/null \ + && curl --fail --silent --show-error "${BASE_URL}/ready" >/dev/null; then + jq -n \ + --arg url "${BASE_URL}" \ + --arg sha "${GITHUB_SHA}" \ + '{status: "healthy", url: $url, image_tag: $sha, commit: $sha}' \ + > release-summary.json exit 0 fi sleep 5 done - echo "Post-deploy health verification failed." >&2 + echo "Post-deploy verification failed: expected the new build SHA and a ready service." >&2 exit 1 - name: Roll back services after deployment failure diff --git a/Dockerfile b/Dockerfile index 8abd169..9f88997 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,8 @@ COPY src ./src COPY scripts ./scripts ARG MODELFORGE_EXTRAS="" +ARG MODELFORGE_BUILD_SHA="unknown" +ENV MODELFORGE_BUILD_SHA="${MODELFORGE_BUILD_SHA}" RUN python -m pip install --no-cache-dir --upgrade pip \ && if [ -n "$MODELFORGE_EXTRAS" ]; then \ python -m pip install --no-cache-dir ".[$MODELFORGE_EXTRAS]"; \ diff --git a/docs/cloud-deployment.md b/docs/cloud-deployment.md index 6b2e837..2d5f78e 100644 --- a/docs/cloud-deployment.md +++ b/docs/cloud-deployment.md @@ -109,7 +109,7 @@ This keeps framework credentials out of runtime JSON configuration and gives fut 4. Run `modelforge config-check` using the production environment. 5. Run `alembic upgrade head` once as a release task. 6. Start the API and external runtime services. -7. Verify `/health` and `/ready`. +7. Verify `/health` reports the released build SHA and `/ready` succeeds. 8. Verify OIDC login and workspace creation. 9. Create a workspace API key and exercise the CLI. 10. Register a model, create a canary, run inference, promote, and roll back. diff --git a/infra/aws/README.md b/infra/aws/README.md index b5082af..05215d2 100644 --- a/infra/aws/README.md +++ b/infra/aws/README.md @@ -123,6 +123,7 @@ TAG="YOUR_GIT_SHA" docker buildx build \ --platform linux/amd64 \ + --build-arg "MODELFORGE_BUILD_SHA=$TAG" \ -t "$API_REPO:$TAG" \ --push \ ../.. @@ -242,13 +243,17 @@ It then: 4. runs the migration task and requires exit code 0; 5. deploys the runtime and waits for ECS stability; 6. deploys the API and waits for ECS stability; -7. verifies public `/health` and `/ready`; +7. verifies public `/health` reports the exact released commit SHA and + `/ready` succeeds; 8. rolls services back to their previous task definitions if deployment or verification fails. A release can be rerun safely after a transient failure: ECR repositories are immutable, and the workflow reuses images already published for the same commit -SHA instead of attempting to overwrite them. Automated releases also fail +SHA instead of attempting to overwrite them. Manually published API images must +set `MODELFORGE_BUILD_SHA` to the same commit SHA when built; a healthy service +with a different or unknown build SHA fails verification and triggers rollback. +Automated releases also fail closed when the API/runtime ECS services still have desired count zero; perform the documented initial `activate_services=true` Terraform apply first. @@ -265,12 +270,17 @@ Terraform then creates `github_actions_deploy_role_arn`. Its trust policy is restricted to: ```text -repo:michaelbawuah/ModelForge:environment:production +repo:michaelbawuah@273423923/ModelForge@1373471661:environment:production ``` -If this repository is forked or renamed, override `github_repository`. If a -different protected GitHub environment is used, override -`github_environment`. +GitHub repositories created after July 15, 2026 use immutable owner and +repository IDs in OIDC subjects. This repository was created after that date, +so the trust policy includes its IDs; the name-only subject would prevent the +release workflow from assuming the role. If this repository is forked or +transferred, set `github_repository`, `github_repository_owner_id`, and +`github_repository_id` to the new repository's name and IDs. For a rename, +update `github_repository`; its IDs remain the same. If a different protected +GitHub environment is used, override `github_environment`. The deploy policy grants only the release-plane permissions required for the two ECR repositories, ECS service/task operations, and passing the existing diff --git a/infra/aws/github-actions.tf b/infra/aws/github-actions.tf index 211d90f..7510369 100644 --- a/infra/aws/github-actions.tf +++ b/infra/aws/github-actions.tf @@ -19,7 +19,7 @@ data "aws_iam_policy_document" "github_actions_assume" { test = "StringEquals" variable = "token.actions.githubusercontent.com:sub" values = [ - "repo:${var.github_repository}:environment:${var.github_environment}", + "repo:${split("/", var.github_repository)[0]}@${var.github_repository_owner_id}/${split("/", var.github_repository)[1]}@${var.github_repository_id}:environment:${var.github_environment}", ] } } diff --git a/infra/aws/terraform.tfvars.example b/infra/aws/terraform.tfvars.example index 50af0ac..a0905bb 100644 --- a/infra/aws/terraform.tfvars.example +++ b/infra/aws/terraform.tfvars.example @@ -16,6 +16,12 @@ oidc_authorization_url = "https://YOUR_IDP/authorize" oidc_token_url = "https://YOUR_IDP/oauth/token" oidc_client_id = "YOUR_CLIENT_ID" +# Optional GitHub Actions release role: use the existing AWS IAM provider ARN +# for https://token.actions.githubusercontent.com. +# github_oidc_provider_arn = "arn:aws:iam::123456789012:oidc-provider/token.actions.githubusercontent.com" +# For a fork or transfer, also set github_repository, +# github_repository_owner_id, and github_repository_id. + # Set true for providers requiring a confidential client secret. oidc_client_secret_required = false diff --git a/infra/aws/variables.tf b/infra/aws/variables.tf index c3e494d..aa31389 100644 --- a/infra/aws/variables.tf +++ b/infra/aws/variables.tf @@ -224,6 +224,18 @@ variable "github_repository" { default = "michaelbawuah/ModelForge" } +variable "github_repository_owner_id" { + description = "Immutable GitHub owner ID for the deploy role's OIDC subject. Update for a different owner." + type = string + default = "273423923" +} + +variable "github_repository_id" { + description = "Immutable GitHub repository ID for the deploy role's OIDC subject. Update for a fork." + type = string + default = "1373471661" +} + variable "github_environment" { description = "GitHub environment encoded into the OIDC subject for production releases." type = string diff --git a/src/modelforge/api/app.py b/src/modelforge/api/app.py index 57a6b41..37a6d91 100644 --- a/src/modelforge/api/app.py +++ b/src/modelforge/api/app.py @@ -1,5 +1,7 @@ """HTTP entry point for the ModelForge control plane.""" +import os + from fastapi import FastAPI from modelforge.api.browser_auth import router as browser_auth_router @@ -35,6 +37,7 @@ async def health() -> dict[str, str]: return { "status": "healthy", "service": "modelforge-api", + "build_sha": os.getenv("MODELFORGE_BUILD_SHA", "unknown"), } app.include_router(dashboard_router) diff --git a/tests/test_api.py b/tests/test_api.py index 0f46208..f16d6b2 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -15,6 +15,7 @@ def test_health() -> None: assert response.json() == { "status": "healthy", "service": "modelforge-api", + "build_sha": "unknown", } @@ -172,4 +173,4 @@ def test_upload_artifact_for_missing_model(tmp_path) -> None: assert response.status_code == 404 assert not (tmp_path / "artifacts").exists() finally: - app.dependency_overrides.pop(get_artifact_store, None) \ No newline at end of file + app.dependency_overrides.pop(get_artifact_store, None) diff --git a/tests/test_system_api.py b/tests/test_system_api.py index 353b2b3..63a6b27 100644 --- a/tests/test_system_api.py +++ b/tests/test_system_api.py @@ -7,10 +7,12 @@ client = TestClient(app) -def test_health_reports_process_alive() -> None: +def test_health_reports_process_alive(monkeypatch) -> None: + monkeypatch.setenv("MODELFORGE_BUILD_SHA", "a" * 40) response = client.get("/health") assert response.status_code == 200 assert response.json()["status"] == "healthy" + assert response.json()["build_sha"] == "a" * 40 def test_readiness_checks_database() -> None: