From b02fb3df6f92e124becb9c1a405f40fb557f2885 Mon Sep 17 00:00:00 2001 From: marcelopenas Date: Mon, 9 Nov 2026 09:08:53 -0300 Subject: [PATCH] chore: retain and correct platform infrastructure changes --- .env.example | 27 +- .github/workflows/ci.yml | 14 +- .github/workflows/deploy-eks.yml | 25 +- README.md | 79 +++-- compose.yml | 111 ++++++- docs/development/get_started.md | 41 ++- docs/infrastructure/deployment.md | 68 ++++- docs/operations/bottlenecks.md | 93 ++++-- docs/operations/load-testing.md | 107 ++++++- k8s/00-namespace-config.yaml | 20 ++ k8s/05-redis.yaml | 40 +++ k8s/10-database.yaml | 52 ++++ k8s/20-account.yaml | 65 ++++ k8s/30-auth.yaml | 49 +++ k8s/40-order.yaml | 65 ++++ k8s/50-gateway.yaml | 49 +++ k8s/60-ingress.yaml | 19 ++ k8s/70-hpa.yaml | 99 ++++++ k8s/71-pdb.yaml | 43 +++ k8s/README.md | 88 +++++- k8s/platform.yaml | 286 ------------------ mocks/exchange-provider/Dockerfile | 7 + mocks/exchange-provider/server.py | 60 ++++ .../infrastructure/test_exchange_provider.py | 56 ++++ tests/k6/platform.js | 38 +++ 25 files changed, 1224 insertions(+), 377 deletions(-) create mode 100644 k8s/00-namespace-config.yaml create mode 100644 k8s/05-redis.yaml create mode 100644 k8s/10-database.yaml create mode 100644 k8s/20-account.yaml create mode 100644 k8s/30-auth.yaml create mode 100644 k8s/40-order.yaml create mode 100644 k8s/50-gateway.yaml create mode 100644 k8s/60-ingress.yaml create mode 100644 k8s/70-hpa.yaml create mode 100644 k8s/71-pdb.yaml delete mode 100644 k8s/platform.yaml create mode 100644 mocks/exchange-provider/Dockerfile create mode 100644 mocks/exchange-provider/server.py create mode 100644 tests/infrastructure/test_exchange_provider.py create mode 100644 tests/k6/platform.js diff --git a/.env.example b/.env.example index a2a3ef0..87ea84b 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,4 @@ -# Local Compose defaults. Never commit the real .env file or production values. +# Local Compose configuration. Copy to .env; never commit the real .env file. POSTGRES_USER=admin POSTGRES_PASSWORD=replace-with-a-local-password POSTGRES_DB=platform @@ -6,17 +6,24 @@ JWT_SECRET=replace-with-at-least-32-random-characters INTERNAL_SERVICE_SECRET=replace-with-a-random-service-secret ACCESS_TOKEN_EXPIRATION_SECONDS=900 DISPLAY_TIME_ZONE=UTC +# Services receive CACHE_REDIS_URL; set this to Redis Cloud or another +# reachable Redis deployment when local Redis is not used. +CACHE_REDIS_URL=redis://redis:6379 +# Optional: defaults to the local Compose mock. Set both for a real provider. +EXCHANGE_PROVIDER_URL=http://exchange-provider-mock:8080 +EXCHANGE_PROVIDER_API_KEY=local-development -# Optional immutable images for local Compose. Leave unset to use latest. -ACCOUNT_IMAGE=ghcr.io/microsserv52/account:latest -AUTH_IMAGE=ghcr.io/microsserv52/auth:latest -GATEWAY_IMAGE=ghcr.io/microsserv52/gateway:latest -ORDER_IMAGE=ghcr.io/microsserv52/order-service:latest +# Product and Exchange are external service images and must be available locally +# or in a registry accessible by Docker. +ACCOUNT_IMAGE=microsserv52-account:local +AUTH_IMAGE=microsserv52-auth:local +EXCHANGE_IMAGE=microsserv52-exchange:local +GATEWAY_IMAGE=microsserv52-gateway:local +ORDER_IMAGE=microsserv52-order:local +PRODUCT_IMAGE=microsserv52-product:local -# AWS/EKS deployment secrets (configure these as GitHub Actions secrets). +# Production EKS values are configured as GitHub Actions secrets/variables, +# not in the local .env file. AWS_REGION=us-east-1 EKS_CLUSTER_NAME=platform ECR_REGISTRY=123456789012.dkr.ecr.us-east-1.amazonaws.com -SPRING_DATASOURCE_URL=jdbc:postgresql://your-rds-endpoint:5432/platform -SPRING_DATASOURCE_USERNAME=platform -SPRING_DATASOURCE_PASSWORD=replace-in-github-secrets diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65030f6..a212e9c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,9 +17,19 @@ jobs: - uses: actions/checkout@v7 - uses: actions/setup-python@v6 with: - python-version: '3.13' + python-version: "3.13" - run: python -m unittest discover -s tests/frontend -p 'test_*.py' + infrastructure: + name: Infrastructure mock contracts + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + - run: python -m unittest discover -s tests/infrastructure -p 'test_*.py' + java: name: ${{ matrix.service }} tests and coverage runs-on: ubuntu-latest @@ -42,7 +52,7 @@ jobs: - uses: actions/setup-java@v5 with: distribution: temurin - java-version: '25' + java-version: "25" cache: maven - name: Install reusable Order contract if: matrix.service == 'order-service' diff --git a/.github/workflows/deploy-eks.yml b/.github/workflows/deploy-eks.yml index 6a0ad13..1d56a32 100644 --- a/.github/workflows/deploy-eks.yml +++ b/.github/workflows/deploy-eks.yml @@ -21,10 +21,11 @@ jobs: - uses: actions/setup-java@v5 with: distribution: temurin - java-version: '25' + java-version: "25" cache: maven - name: Verify before deployment run: | + (cd api/order && ./mvnw --batch-mode --no-transfer-progress install -DskipTests -Dspring-boot.repackage.skip=true) for service in account auth gateway order order-service; do (cd "api/$service" && ./mvnw --batch-mode --no-transfer-progress verify) done @@ -59,21 +60,39 @@ jobs: POSTGRES_PASSWORD: ${{ secrets.SPRING_DATASOURCE_PASSWORD }} SPRING_DATASOURCE_URL: ${{ secrets.SPRING_DATASOURCE_URL }} SPRING_DATASOURCE_USERNAME: ${{ secrets.SPRING_DATASOURCE_USERNAME }} + PRODUCT_URL: ${{ vars.PRODUCT_URL }} + EXCHANGE_URL: ${{ vars.EXCHANGE_URL }} + CACHE_REDIS_URL: ${{ secrets.CACHE_REDIS_URL }} run: | + test -n "$PRODUCT_URL" || { echo "PRODUCT_URL repository variable is required"; exit 1; } + test -n "$EXCHANGE_URL" || { echo "EXCHANGE_URL repository variable is required"; exit 1; } + test -n "$CACHE_REDIS_URL" || { echo "CACHE_REDIS_URL repository secret is required"; exit 1; } aws eks update-kubeconfig --region "$AWS_REGION" --name "$EKS_CLUSTER_NAME" kubectl create namespace platform --dry-run=client -o yaml | kubectl apply -f - kubectl -n platform create secret generic platform-secrets \ --from-literal=JWT_SECRET="$JWT_SECRET" \ --from-literal=INTERNAL_SERVICE_SECRET="$INTERNAL_SERVICE_SECRET" \ + --from-literal=CACHE_REDIS_URL="$CACHE_REDIS_URL" \ --from-literal=POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ --from-literal=SPRING_DATASOURCE_URL="$SPRING_DATASOURCE_URL" \ --from-literal=SPRING_DATASOURCE_USERNAME="$SPRING_DATASOURCE_USERNAME" \ --dry-run=client -o yaml | kubectl apply -f - - kubectl apply -f k8s/platform.yaml + kubectl apply -f k8s/00-namespace-config.yaml + jq -n --arg product "$PRODUCT_URL" --arg exchange "$EXCHANGE_URL" \ + '{data: {PRODUCT_URL: $product, EXCHANGE_URL: $exchange}}' > "$RUNNER_TEMP/platform-config-patch.json" + kubectl -n platform patch configmap platform-config --type merge \ + --patch-file "$RUNNER_TEMP/platform-config-patch.json" + kubectl apply -f k8s/20-account.yaml kubectl -n platform set image deployment/account account="$REGISTRY/account:$TAG" + kubectl apply -f k8s/30-auth.yaml kubectl -n platform set image deployment/auth auth="$REGISTRY/auth:$TAG" - kubectl -n platform set image deployment/gateway gateway="$REGISTRY/gateway:$TAG" + kubectl apply -f k8s/40-order.yaml kubectl -n platform set image deployment/order order="$REGISTRY/order:$TAG" + kubectl apply -f k8s/50-gateway.yaml + kubectl -n platform set image deployment/gateway gateway="$REGISTRY/gateway:$TAG" + kubectl apply -f k8s/60-ingress.yaml + kubectl apply -f k8s/70-hpa.yaml + kubectl apply -f k8s/71-pdb.yaml kubectl -n platform rollout status deployment/account --timeout=180s kubectl -n platform rollout status deployment/auth --timeout=180s kubectl -n platform rollout status deployment/gateway --timeout=180s diff --git a/README.md b/README.md index 75d3034..0dc54a8 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,43 @@ Open `http://localhost:8081/`, `/store/` or `/admin/`. This separate local-only mode visibly labels disposable API fixtures. See the [local testing guide](docs/frontend/delivery.md) for fixture inputs, real-service mode and verification limits. +## Which environment should I run? + +| Environment | Purpose | Database | Exchange provider | Entry point | +| -------------- | ----------------------------------------- | ----------------------- | ----------------------------------- | -------------------------------- | +| Docker Compose | Local development and integration testing | PostgreSQL container | Local deterministic mock by default | `http://localhost:8080` | +| MicroK8s | Local Kubernetes validation | `k8s/10-database.yaml` | External Exchange service URL | Ingress address | +| AWS EKS | Production backend | External PostgreSQL/RDS | External Exchange service URL | Production Ingress/load balancer | + +Compose and Kubernetes are separate runtime environments. Do not use the +Compose mock or its local defaults in EKS. + +### Run local Compose + +```bash +cp .env.example .env +docker compose up --build -d +docker compose ps +docker compose down +``` + +Product and Exchange images must be available locally or in a registry because +their source repositories are not included here. Compose supplies PostgreSQL, +the checked-out Java services, and the local Exchange provider mock. + +### Run local MicroK8s + +Follow the [Kubernetes runbook](k8s/README.md). It requires MicroK8s with the +Nginx ingress addon, a `platform-secrets` Secret, reachable Product/Exchange +services, and local images loaded into MicroK8s. + +### Run production EKS + +Configure the required GitHub Actions secrets and `PRODUCT_URL`/ +`EXCHANGE_URL` repository variables, then run the manual **Deploy to AWS EKS** +workflow. EKS uses the production database URL and does not deploy the local +PostgreSQL manifest. + ## CI/CD and environment variables Each service repository owns lint, format, tests, JaCoCo quality gates, CodeQL, @@ -58,28 +95,32 @@ real `.env` file or any secret values. Configure repository secrets in **Settings → Secrets and variables → Actions** for each repository: -| Repository | Required secrets | -| --- | --- | -| `Microsserv52/platform` | `AWS_REGION`, `AWS_DEPLOY_ROLE_ARN`, `EKS_CLUSTER_NAME`, `JWT_SECRET`, `INTERNAL_SERVICE_SECRET`, `SPRING_DATASOURCE_URL`, `SPRING_DATASOURCE_USERNAME`, `SPRING_DATASOURCE_PASSWORD` | -| `Microsserv52/account` | `JWT_SECRET`, `INTERNAL_SERVICE_SECRET`, `SPRING_DATASOURCE_URL`, `SPRING_DATASOURCE_USERNAME`, `SPRING_DATASOURCE_PASSWORD` | -| `Microsserv52/auth` | `JWT_SECRET`, `INTERNAL_SERVICE_SECRET`, `ACCOUNT_URL`, `ACCESS_TOKEN_EXPIRATION_SECONDS` | -| `Microsserv52/gateway` | `JWT_SECRET`, `AUTH_URL`, `ACCOUNT_URL`, `ORDER_URL`, `PRODUCT_URL`, `EXCHANGE_URL` | -| `Microsserv52/order-service` | `JWT_SECRET`, `SPRING_DATASOURCE_URL`, `SPRING_DATASOURCE_USERNAME`, `SPRING_DATASOURCE_PASSWORD`, `PRODUCT_URL` | -| `Microsserv52/order` | No runtime secrets; this repository contains the reusable API contract. | - -For local Compose, put the values in the root `.env`. For MicroK8s, create -the `platform-secrets` Kubernetes secret as described in -[`k8s/README.md`](k8s/README.md). For EKS, use an RDS PostgreSQL URL and add -the values to the **platform** repository's Actions secrets. +| Repository | Required secrets | +| ---------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Microsserv52/platform` | `AWS_REGION`, `AWS_DEPLOY_ROLE_ARN`, `EKS_CLUSTER_NAME`, `JWT_SECRET`, `INTERNAL_SERVICE_SECRET`, `SPRING_DATASOURCE_URL`, `SPRING_DATASOURCE_USERNAME`, `SPRING_DATASOURCE_PASSWORD`, `CACHE_REDIS_URL` | +| `Microsserv52/account` | `JWT_SECRET`, `INTERNAL_SERVICE_SECRET`, `SPRING_DATASOURCE_URL`, `SPRING_DATASOURCE_USERNAME`, `SPRING_DATASOURCE_PASSWORD` | +| `Microsserv52/auth` | `JWT_SECRET`, `INTERNAL_SERVICE_SECRET`, `ACCOUNT_URL`, `ACCESS_TOKEN_EXPIRATION_SECONDS` | +| `Microsserv52/gateway` | `JWT_SECRET`, `AUTH_URL`, `ACCOUNT_URL`, `ORDER_URL`, `PRODUCT_URL`, `EXCHANGE_URL` | +| `Microsserv52/order-service` | `JWT_SECRET`, `SPRING_DATASOURCE_URL`, `SPRING_DATASOURCE_USERNAME`, `SPRING_DATASOURCE_PASSWORD`, `PRODUCT_URL` | +| `Microsserv52/order` | No runtime secrets; this repository contains the reusable API contract. | + +For local Compose, copy `.env.example` to `.env`; Compose provisions the local +PostgreSQL database and defaults internal service settings. Product and +Exchange images plus Exchange provider settings are the only external local +inputs. For MicroK8s, create the `platform-secrets` Kubernetes secret as +described in [`k8s/README.md`](k8s/README.md). For EKS, use an RDS PostgreSQL +URL, add secret values to the **platform** repository's Actions secrets, and +configure `PRODUCT_URL` and `EXCHANGE_URL` as Actions variables and +`CACHE_REDIS_URL` as an Actions secret. ### Environment variable locations -| Variable type | Local development | GitHub Actions | Kubernetes/EKS | -| --- | --- | --- | --- | -| Service URLs and image tags | Root `.env` | Repository variables or workflow defaults | ConfigMap/deployment values | -| JWT and internal credentials | Root `.env` (never commit) | Repository **Secrets** | `platform-secrets` | -| Database URL and credentials | Root `.env` | Platform or service **Secrets** | `platform-secrets` / AWS Secrets Manager | -| AWS deployment values | Not required | Platform **Secrets** (`AWS_REGION`, `AWS_DEPLOY_ROLE_ARN`, `EKS_CLUSTER_NAME`) | IAM/OIDC and cluster configuration | +| Variable type | Local development | GitHub Actions | Kubernetes/EKS | +| ---------------------------- | -------------------------- | ------------------------------------------------------------------------------ | ---------------------------------------- | +| Service URLs and image tags | Root `.env` | Repository variables or workflow defaults | ConfigMap/deployment values | +| JWT and internal credentials | Root `.env` (never commit) | Repository **Secrets** | `platform-secrets` | +| Database URL and credentials | Root `.env` | Platform or service **Secrets** | `platform-secrets` / AWS Secrets Manager | +| AWS deployment values | Not required | Platform **Secrets** (`AWS_REGION`, `AWS_DEPLOY_ROLE_ARN`, `EKS_CLUSTER_NAME`) | IAM/OIDC and cluster configuration | The service workflows need `packages: write` to publish GHCR images. The platform workflow needs `packages: read` to pull them. Keep GHCR packages diff --git a/compose.yml b/compose.yml index e09ba26..97ff014 100644 --- a/compose.yml +++ b/compose.yml @@ -2,61 +2,132 @@ name: platform services: database: - image: Microsserv52-postgres:latest + image: postgres:17-alpine hostname: database environment: - - POSTGRES_USER=admin - - POSTGRES_PASSWORD=admin - - POSTGRES_DB=platform + - POSTGRES_USER=${POSTGRES_USER:-admin} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-admin} + - POSTGRES_DB=${POSTGRES_DB:-platform} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 5s + timeout: 5s + retries: 10 + + redis: + image: redis:7.4-alpine + hostname: redis + command: ["redis-server", "--appendonly", "yes"] + volumes: + - redis-data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 product: - image: Microsserv52-product:latest + image: ${PRODUCT_IMAGE:-microsserv52-product:local} hostname: product environment: - LOG_LEVEL=info + - CACHE_REDIS_URL=${CACHE_REDIS_URL:-redis://redis:6379} deploy: replicas: 3 depends_on: - - database + database: + condition: service_healthy + redis: + condition: service_healthy order: - image: ${ORDER_IMAGE:-ghcr.io/microsserv52/order-service:latest} + image: ${ORDER_IMAGE:-microsserv52-order:local} + build: + context: . + dockerfile: api/order-service/Dockerfile hostname: order environment: - LOG_LEVEL=info + - SPRING_DATASOURCE_URL=jdbc:postgresql://database:5432/${POSTGRES_DB:-platform} + - SPRING_DATASOURCE_USERNAME=${POSTGRES_USER:-admin} + - SPRING_DATASOURCE_PASSWORD=${POSTGRES_PASSWORD:-admin} + - PRODUCT_URL=http://product:8080 + - JWT_SECRET=${JWT_SECRET:-change-me-in-development-use-a-long-random-secret} + - CACHE_REDIS_URL=${CACHE_REDIS_URL:-redis://redis:6379} deploy: replicas: 3 depends_on: - - database - - product + database: + condition: service_healthy + product: + condition: service_started + redis: + condition: service_healthy exchange: - image: Microsserv52-exchange:latest + image: ${EXCHANGE_IMAGE:-microsserv52-exchange:local} hostname: exchange environment: - LOG_LEVEL=info + - EXCHANGE_PROVIDER_URL=${EXCHANGE_PROVIDER_URL:-http://exchange-provider-mock:8080} + - EXCHANGE_PROVIDER_API_KEY=${EXCHANGE_PROVIDER_API_KEY:-local-development} + - CACHE_REDIS_URL=${CACHE_REDIS_URL:-redis://redis:6379} deploy: replicas: 3 + depends_on: + exchange-provider-mock: + condition: service_healthy + redis: + condition: service_healthy + + exchange-provider-mock: + build: + context: ./mocks/exchange-provider + hostname: exchange-provider-mock + healthcheck: + test: + [ + "CMD", + "python", + "-c", + "import urllib.request; urllib.request.urlopen('http://localhost:8080/health')", + ] + interval: 5s + timeout: 3s + retries: 10 account: - image: ${ACCOUNT_IMAGE:-ghcr.io/microsserv52/account:latest} + image: ${ACCOUNT_IMAGE:-microsserv52-account:local} + build: + context: ./api/account hostname: account environment: - LOG_LEVEL=info - JWT_SECRET=${JWT_SECRET:-change-me-in-development-use-a-long-random-secret} - INTERNAL_SERVICE_SECRET=${INTERNAL_SERVICE_SECRET:-change-me-in-development} + - SPRING_DATASOURCE_URL=jdbc:postgresql://database:5432/${POSTGRES_DB:-platform} + - SPRING_DATASOURCE_USERNAME=${POSTGRES_USER:-admin} + - SPRING_DATASOURCE_PASSWORD=${POSTGRES_PASSWORD:-admin} + - CACHE_REDIS_URL=${CACHE_REDIS_URL:-redis://redis:6379} deploy: replicas: 3 depends_on: - - database + database: + condition: service_healthy + redis: + condition: service_healthy auth: - image: ${AUTH_IMAGE:-ghcr.io/microsserv52/auth:latest} + image: ${AUTH_IMAGE:-microsserv52-auth:local} + build: + context: ./api/auth hostname: auth environment: - LOG_LEVEL=info - JWT_SECRET=${JWT_SECRET:-change-me-in-development-use-a-long-random-secret} - INTERNAL_SERVICE_SECRET=${INTERNAL_SERVICE_SECRET:-change-me-in-development} + - ACCESS_TOKEN_EXPIRATION_SECONDS=${ACCESS_TOKEN_EXPIRATION_SECONDS:-900} + - CACHE_REDIS_URL=${CACHE_REDIS_URL:-redis://redis:6379} deploy: replicas: 3 depends_on: @@ -64,18 +135,27 @@ services: - account gateway: - image: ${GATEWAY_IMAGE:-ghcr.io/microsserv52/gateway:latest} + image: ${GATEWAY_IMAGE:-microsserv52-gateway:local} + build: + context: ./api/gateway hostname: gateway environment: - LOG_LEVEL=info - JWT_SECRET=${JWT_SECRET:-change-me-in-development-use-a-long-random-secret} + - AUTH_URL=http://auth:8080 + - ACCOUNT_URL=http://account:8080 + - ORDER_URL=http://order:8080 + - PRODUCT_URL=http://product:8080 + - EXCHANGE_URL=http://exchange:8080 - SESSION_COOKIE_SECURE=false - SESSION_COOKIE_PATH=/api + - CACHE_REDIS_URL=${CACHE_REDIS_URL:-redis://redis:6379} deploy: replicas: 3 depends_on: - account - auth + - redis landing: image: microsserv52-landing:local @@ -116,3 +196,6 @@ services: - landing - store - admin + +volumes: + redis-data: diff --git a/docs/development/get_started.md b/docs/development/get_started.md index 3255b0e..ab9589a 100644 --- a/docs/development/get_started.md +++ b/docs/development/get_started.md @@ -35,20 +35,48 @@ Open the address printed by Zensical, normally ## Run the platform -The root `compose.yml` describes the intended local topology. Build or publish -the service images referenced there, then start the stack with: +Choose one runtime mode: + +- Use **Docker Compose** for local development and integration testing. +- Use **MicroK8s** to validate the Kubernetes deployment locally. +- Use **AWS EKS** for the production backend. + +These modes do not share runtime resources or defaults. + +### Docker Compose (local) + +The root `compose.yml` is for local testing only. It starts PostgreSQL with +local defaults and builds Account, Auth, Gateway and Order from their checked +out submodules. Product and Exchange are external service repositories, so +their images must be available locally or in a registry. Compose includes a +local deterministic Exchange provider mock by default: ```bash +test -f .env || cp .env.example .env docker compose up -d docker compose ps docker compose logs -f gateway ``` +No external Exchange provider configuration is required for local development. +To use a real provider instead, set `EXCHANGE_PROVIDER_URL` and +`EXCHANGE_PROVIDER_API_KEY` in `.env`. + +The mock responds to `/health`, `/latest`, `/v1/latest`, and `/latest.json` +with deterministic USD, EUR, BRL, GBP and JPY rates. It is intended for local +testing only and must not be used by production workloads. + +The Product and Exchange images are required artifacts, but do not require +additional configuration when they are already available locally or through +Docker's configured registry. +PostgreSQL credentials, internal service URLs, JWT settings and the internal +service secret have local defaults. + The local edge is Nginx on `http://localhost:8080`. Service containers are replicated in the Compose topology and the gateway remains the only application entry point exposed through the proxy. -For MicroK8s or EKS, see the [Kubernetes deployment guide](https://github.com/Microsserv52/platform/blob/main/k8s/README.md). +For MicroK8s or EKS, see the [Kubernetes deployment guide](../infrastructure/eks.md). The gateway is exposed on port `8080` by the compose configuration. The repository snapshot currently contains the Order API and documentation @@ -56,6 +84,13 @@ submodules, while the Product, Exchange, Account, Auth and Gateway images must be supplied by their respective service repositories before the full stack can start. +### Kubernetes + +For local MicroK8s or production EKS, follow the +[Kubernetes deployment guide](../infrastructure/eks.md). MicroK8s and EKS require +Kubernetes Secrets and reachable Product/Exchange services; they do not use +the Compose `.env` file. + ## Verify a service ```bash diff --git a/docs/infrastructure/deployment.md b/docs/infrastructure/deployment.md index 81dd234..d256180 100644 --- a/docs/infrastructure/deployment.md +++ b/docs/infrastructure/deployment.md @@ -19,9 +19,25 @@ uv run zensical build --clean ## Application deployment -The application topology is declared in [`compose.yml`](../../compose.yml). The -stack contains PostgreSQL, Product, Order, Exchange, Account, Auth, Gateway -and Nginx. Nginx listens on port `8080` and forwards traffic to the gateway. +The repository supports three distinct runtime modes: + +1. **Docker Compose** — local development and integration testing. It + provisions PostgreSQL and uses the local Exchange provider mock. +2. **MicroK8s** — local Kubernetes validation. It uses the Kubernetes + manifests and can deploy the local PostgreSQL manifest. +3. **AWS EKS** — production. It uses external PostgreSQL/RDS, production + secrets, external Product/Exchange URLs, and the manual deployment workflow. + +The local application topology is declared in [`compose.yml`](../../compose.yml). +Nginx listens on port `8080` and forwards traffic to the gateway. Run it with: + +```bash +cp .env.example .env +docker compose up --build -d +docker compose ps +``` + +Stop it with `docker compose down`. For a deployment environment: @@ -32,14 +48,50 @@ For a deployment environment: 4. Apply migrations before starting dependent services. 5. Verify health endpoints and an authenticated product/order flow. -The compose file is a local topology description, not a production -orchestration manifest. Production deployment must additionally define -resource limits, replicas, persistent storage, TLS, backups, observability, -network policies and a rollback strategy. +The compose file is not a production orchestration manifest. Production uses +the Kubernetes manifests and must additionally define persistent storage, TLS, +backups, observability, network policies and a rollback strategy. The repository includes a manual `Deploy to AWS EKS` workflow. Configure `AWS_REGION`, `AWS_DEPLOY_ROLE_ARN`, `EKS_CLUSTER_NAME`, `JWT_SECRET`, `INTERNAL_SERVICE_SECRET`, `SPRING_DATASOURCE_URL`, `SPRING_DATASOURCE_USERNAME` and `SPRING_DATASOURCE_PASSWORD` as GitHub -Actions secrets. The AWS role should use OIDC and least-privilege ECR/EKS +Actions secrets. Also set `CACHE_REDIS_URL` as a secret and `PRODUCT_URL`/ +`EXCHANGE_URL` as repository variables. The AWS role should use OIDC and least-privilege ECR/EKS permissions; no long-lived AWS access key is required. + +## Infrastructure review scope and verification + +The infrastructure review preserves the five earlier Compose, MicroK8s, Redis, +Exchange mock and load-testing commits without republishing the merged frontend +or product-image changes. The split Kubernetes files replace `k8s/platform.yaml`; +the manual EKS workflow applies only backend resources. Frontends remain static +CDN exports, with hosting and environment values to be agreed before deployment. + +Compose builds the checked-out Account, Auth, Gateway and Order sources under +explicit `:local` image tags. Product and Exchange sources are still absent; +provide those images under the configured tags, or use immutable registry image +references in `.env`. The PostgreSQL baseline is explicitly version 17 to match +the local Kubernetes manifest; Redis is constrained to 7.4. Existing database +volumes from another PostgreSQL major require a separate backup/migration plan. +The MicroK8s database and Redis manifests have no persistent volumes and are +disposable. Compose preserves Redis data in a named volume. + +The provider mock uses deterministic USD reference rates and rebases them for +supported base currencies. Unsupported currencies return HTTP 400. Its date is +the current UTC date; the rates are fixtures, not live financial data. Validate +its HTTP behavior without starting the full platform: + +```bash +python3 -m unittest discover -s tests/infrastructure -p 'test_*.py' +python3 -m unittest discover -s tests/frontend -p 'test_*.py' +docker compose config --quiet +uv run zensical build --clean +``` + +The k6 catalog baseline requires HTTP 200 and fails on missing routes or denied +requests. It is not evidence of authenticated checkout capacity. Redis endpoint +wiring does not implement caching; service-owned cache behavior remains future +work. Kubernetes resources and the manual EKS workflow must be validated in the +target cluster before deployment; no cluster or cloud deployment is performed +as part of this review. diff --git a/docs/operations/bottlenecks.md b/docs/operations/bottlenecks.md index 75e1c04..462f509 100644 --- a/docs/operations/bottlenecks.md +++ b/docs/operations/bottlenecks.md @@ -4,19 +4,72 @@ icon: lucide/gauge # Bottlenecks and performance -The course project allocates substantial effort to identifying and addressing -bottlenecks. Performance work must be evidence-based: measure a baseline, -change one relevant variable, then compare the result. +This study describes where the current platform can become constrained and +what has been done about each constraint. It is an architecture assessment, +not a measured capacity claim: no representative production load-test result +is recorded yet. A bottleneck is acceptable when its capacity is demonstrably +above the system's required peak, even if it remains the narrowest part of the +request path. -## Candidate bottlenecks +## Status definitions -- Gateway connection limits and request routing; -- synchronous Order-to-Product calls; -- database indexes, connection pools and lock contention; -- Exchange provider latency, quotas and availability; -- cold starts and insufficient pod resources; -- network traffic between availability zones; -- serialization, logging and excessive payload size. +| Status | Meaning | +| --------------- | ------------------------------------------------------------------------------------ | +| **Mitigated** | A control is present and the risk is bounded, but it still needs load-test evidence. | +| **Residual** | The risk is reduced, but the component remains a likely limiting resource. | +| **Unmitigated** | The architecture or deployment currently has no control for the risk. | +| **External** | The limit is owned by a dependency and cannot be removed by this repository. | + +## Current bottleneck study + +| Bottleneck | Why it can limit the platform | Current mitigation | Status | What must be verified | +| --------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Public Ingress/Gateway path | Every request enters through the edge and Gateway; connection queues, TLS and routing work are concentrated there. | Gateway has three replicas in Kubernetes; only Gateway is public; resource requests/limits are defined. | **Residual** | p95/p99 latency, active connections, 5xx rate and ingress/controller throughput during peak traffic. | +| Synchronous request chain | Order creation depends on Product, and currency conversion depends on Exchange. One slow dependency increases end-to-end latency. | Services are separated and independently replicated; dependency latency is observable by design. Redis is now available as a shared cache endpoint, but service-level cache logic is not implemented in this repository. | **Residual** | Trace `POST /orders` and converted-order reads; add cache-aside policies only for safe read data; set and test timeouts, bounded retries and failure behavior. | +| External Exchange provider | Provider quotas, latency, outages and rate freshness directly affect conversion requests. | Local Compose uses a deterministic mock; production keeps the provider behind Exchange rather than exposing it directly. Redis can store short-lived rates after Exchange implements cache-aside behavior. | **External** | Provider quota, timeout, error rate and fallback/cache policy. A provider outage cannot be eliminated here; the user-facing degraded behavior must be explicit. | +| PostgreSQL persistence | Connection pools, locks, indexes, I/O and a single writer can serialize account, product and order work. | Kubernetes applications use three replicas and resource bounds; EKS is expected to use managed PostgreSQL/RDS. | **Residual** | Pool saturation, query latency, lock waits, CPU, I/O, storage growth, backup and failover behavior. | +| Database availability | The local `10-database.yaml` has one PostgreSQL replica and no persistent volume, so a pod loss loses availability and can lose local data. | The manifest is documented as MicroK8s-only; EKS skips it and uses external PostgreSQL/RDS. | **Unmitigated locally / mitigated by production boundary** | Use persistent storage and backups for MicroK8s, or verify RDS Multi-AZ, backups and recovery objectives for EKS. | +| Application CPU and memory | A container can be throttled or evicted if its limit is too low, or starve neighbors if it has no bound. | All five Kubernetes Deployments define requests of `250m` CPU/`256Mi` memory and limits of `500m` CPU/`512Mi` memory. | **Mitigated** | Compare utilization and throttling against limits; tune values from a representative workload. | +| Replica scaling | Fixed three-replica application Deployments can saturate during a burst and do not automatically add capacity. | HPA manifests now target Gateway, Order, Account and Auth at 60% CPU with 3-10 replicas. | **Mitigated** | Install Metrics Server and use k6 to verify scale-up latency, scale-down stabilization and node capacity. | +| Rolling disruption | A rollout or node drain can remove too much capacity if replicas are unavailable simultaneously. | PDBs now require two available pods for Gateway, Order, Account and Auth. | **Mitigated** | Add topology spread constraints and verify drain/rollout behavior in a non-production namespace. | +| Product and Exchange image availability | These services are external repositories and are not built from this checkout; missing images prevent the local or production topology from starting. | Compose and EKS accept explicit image/URL configuration; the local Exchange provider mock removes the provider dependency, not the Exchange service dependency. | **Residual** | Pin and scan immutable images, verify registry access, and run a pre-deployment dependency check. | +| Cross-zone network traffic | Calls between services or a database in different availability zones add latency and transfer cost. | Internal services remain cluster-internal; EKS is designed for private subnets and multi-AZ placement. | **Residual** | Measure dependency latency by zone and evaluate placement, connection reuse and data-transfer cost. | +| Logging, serialization and payload size | Excessive logs or large payloads consume CPU, network and storage and increase tail latency. | Structured, low-cardinality observability requirements are documented. | **Unmitigated** | Enforce sampling/redaction, payload limits and log-volume alerts; compare overhead under load. | +| Cache correctness and invalidation | Stale product or exchange data can be worse than a slow request; cache stampedes can move load to the database/provider. | Compose and Kubernetes expose `CACHE_REDIS_URL`; cache implementation, TTLs and invalidation remain service-owned. | **Unmitigated** | Define per-resource TTLs, keys, invalidation, stampede protection and hit-rate/error metrics before enabling caching. | + +### Capacity interpretation + +The current Kubernetes requests reserve at least: + +- `3 × 250m = 750m` CPU and `3 × 256Mi = 768Mi` memory per application + Deployment (Account, Auth, Order and Gateway); +- `3.0` CPU cores and `3Gi` memory across those four application + Deployments, before the database, ingress controller, system pods and + external Product/Exchange workloads are included; +- a database request of `250m` CPU and `256Mi` memory in the local-only + PostgreSQL manifest. + +These are scheduling reservations, not measured throughput. The Gateway and +Ingress remain architectural choke points, but that is acceptable only if +measured peak traffic remains below their tested capacity with the required +latency and error-rate headroom. The project must not claim that three +replicas support a user count until the load-test evidence demonstrates it. + +## Required mitigation backlog + +The following work is required before a production capacity claim: + +1. Define peak requests per second, concurrent users, dataset size and SLOs. +2. Run the flows in [Load testing](load-testing.md), including login, product + browsing, order creation and converted-order reads. +3. Run the HPA and k6 procedure in [Load testing](load-testing.md). +4. Configure explicit timeout, bounded retry and degraded-response behavior + for Product, Exchange and the database. +5. Implement service-owned cache-aside behavior and verify hit rate, freshness + and invalidation for each cached resource. +6. Verify PostgreSQL indexes, pool sizes, backups, failover and recovery. +7. Repeat the tests after each change and retain the results with image tags, + cluster size and region. ## Investigation procedure @@ -35,12 +88,12 @@ CPU utilization alone. Each experiment should record: -| Field | Example | -| --- | --- | -| Flow | `POST /orders` | -| Workload | 50 concurrent users, 10 minutes | -| Baseline | p95 800 ms, 2% errors | -| Change | Add product lookup index | -| Result | p95 240 ms, 0% errors | -| Trade-off | Additional storage | -| Evidence | Dashboard, trace or load-test artifact | +| Field | Example | +| --------- | -------------------------------------- | +| Flow | `POST /orders` | +| Workload | 50 concurrent users, 10 minutes | +| Baseline | p95 800 ms, 2% errors | +| Change | Add product lookup index | +| Result | p95 240 ms, 0% errors | +| Trade-off | Additional storage | +| Evidence | Dashboard, trace or load-test artifact | diff --git a/docs/operations/load-testing.md b/docs/operations/load-testing.md index fe93558..0aff163 100644 --- a/docs/operations/load-testing.md +++ b/docs/operations/load-testing.md @@ -21,17 +21,111 @@ Start with representative flows: Define arrival rate or concurrent users, duration, warm-up, dataset and acceptable thresholds before running the test. +For this platform, record the expected peak and a headroom target before the +test. A component may remain the measured bottleneck and still pass if it +supports the required peak with the agreed latency and error-rate margin. The +goal is not to remove every bottleneck; it is to keep every known bottleneck +below the system requirement or make its failure mode safe and observable. + ## Measurements Record throughput, p50/p95/p99 latency, HTTP status distribution, timeouts, dependency latency, CPU, memory, replicas, database connections and provider -errors. Include the exact commit, image tags, cluster size and region. +errors. Also record ingress/controller saturation, CPU throttling, HPA +activity, pod evictions, and the remaining headroom at the narrowest +component. Include the exact commit, image tags, cluster size and region. + +## Read-only k6 catalog baseline + +The repository includes [`tests/k6/platform.js`](../../tests/k6/platform.js). +It exercises the public catalog at `/api/products` through the Compose edge. +Only HTTP 200 counts as success: authentication errors, missing routes and server +errors fail the checks. Use `PATHS=/products` when calling Gateway directly. +This read-only baseline does not verify order creation or authenticated capacity. +`RATE` counts iterations per second; each iteration sends one request per path. + +Install k6 using the official instructions for your operating system, then run +the local Compose baseline: + +```bash +docker compose up --build -d +k6 run \ + --env BASE_URL=http://localhost:8080 \ + --env RATE=10 \ + --env DURATION=2m \ + tests/k6/platform.js +docker compose down +``` + +For an authenticated scenario, extend the script with the API login/session +flow and CSRF handling for writes; do not commit credentials, cookies or tokens. +The default script is intentionally read-only and does not create test data. + +## Kubernetes HPA exercise + +The manifests in `k8s/70-hpa.yaml` configure HPA for Gateway, Order, Account and +Auth with three minimum replicas, ten maximum replicas and a 60% CPU target. +The manifests in `k8s/71-pdb.yaml` preserve two available replicas during +voluntary disruption. MicroK8s supports these manifests, but HPA requires the +cluster to be running and Metrics Server to be enabled: + +```bash +microk8s start +microk8s status --wait-ready +microk8s enable dns metrics-server ingress +microk8s kubectl get nodes +microk8s kubectl top pods -n platform +``` + +Load the application images into MicroK8s or configure registry references +first. Then deploy the local test environment and HPA: + +```bash +microk8s kubectl apply -f k8s/00-namespace-config.yaml +microk8s kubectl apply -f k8s/05-redis.yaml +microk8s kubectl apply -f k8s/10-database.yaml +microk8s kubectl apply -f k8s/20-account.yaml +microk8s kubectl apply -f k8s/30-auth.yaml +microk8s kubectl apply -f k8s/40-order.yaml +microk8s kubectl apply -f k8s/50-gateway.yaml +microk8s kubectl apply -f k8s/60-ingress.yaml +microk8s kubectl apply -f k8s/70-hpa.yaml +microk8s kubectl apply -f k8s/71-pdb.yaml +microk8s kubectl get hpa -n platform --watch +``` + +In a controlled non-production namespace, generate load from the host and +observe scaling: + +```bash +k6 run \ + --env BASE_URL=http:// \ + --env PATHS=/products \ + --env RATE=50 \ + --env DURATION=10m \ + tests/k6/platform.js +microk8s kubectl get deployment gateway -n platform -w +microk8s kubectl get hpa gateway -n platform +``` + +Record initial and peak replica counts, HPA current/desired CPU, scale-up +latency, p95/p99 latency, error rate, pod evictions and node capacity. The +test passes only when the defined peak request rate and latency/error SLOs are +met with headroom; scaling alone is not evidence of sufficient capacity. + +If the ingress addon is not exposing a host address, use port forwarding for a +controlled test: + +```bash +microk8s kubectl -n platform port-forward service/gateway 18080:8080 +k6 run --env BASE_URL=http://127.0.0.1:18080 --env PATHS=/products tests/k6/platform.js +``` ## Kubernetes exercise The course material demonstrates an HPA for Gateway and a temporary in-cluster -load generator. The following is suitable only for a controlled test -namespace: +load generator. Do not create a second HPA when `70-hpa.yaml` is installed. The following is +suitable only for a separate controlled namespace without the managed HPA: ```bash kubectl autoscale deployment gateway --cpu-percent=50 --min=1 --max=10 @@ -39,6 +133,7 @@ kubectl get hpa gateway kubectl get pods -l app=gateway --watch ``` -Use a maintained load-testing tool for formal results. A loop with `wget` is -useful as a smoke experiment but is not sufficient evidence for capacity or -SLA claims. Delete temporary HPAs and load-generator pods after testing. +Use k6 for formal results. A loop with `wget` is useful as a smoke experiment +but is not sufficient evidence for capacity or SLA claims. Delete temporary +HPAs and load-generator pods after testing; keep the managed HPA manifests when +they are part of the production deployment. diff --git a/k8s/00-namespace-config.yaml b/k8s/00-namespace-config.yaml new file mode 100644 index 0000000..60427da --- /dev/null +++ b/k8s/00-namespace-config.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: platform +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: platform-config + namespace: platform +data: + SPRING_DATASOURCE_URL: jdbc:postgresql://database:5432/platform + SPRING_DATASOURCE_USERNAME: admin + ACCOUNT_URL: http://account:8080 + AUTH_URL: http://auth:8080 + ORDER_URL: http://order:8080 + # Replace these with the production Product and Exchange service URLs. + PRODUCT_URL: https://replace-with-product-service-url + EXCHANGE_URL: https://replace-with-exchange-service-url + CACHE_REDIS_URL: redis://redis:6379 diff --git a/k8s/05-redis.yaml b/k8s/05-redis.yaml new file mode 100644 index 0000000..4030bfd --- /dev/null +++ b/k8s/05-redis.yaml @@ -0,0 +1,40 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis + namespace: platform +spec: + selector: + app: redis + ports: + - port: 6379 + targetPort: 6379 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis + namespace: platform +spec: + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + containers: + - name: redis + image: redis:7.4-alpine + args: ["redis-server", "--appendonly", "yes"] + ports: + - containerPort: 6379 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 250m + memory: 256Mi diff --git a/k8s/10-database.yaml b/k8s/10-database.yaml new file mode 100644 index 0000000..4a8bb2b --- /dev/null +++ b/k8s/10-database.yaml @@ -0,0 +1,52 @@ +apiVersion: v1 +kind: Service +metadata: + name: database + namespace: platform +spec: + selector: + app: database + ports: + - port: 5432 + targetPort: 5432 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: database + namespace: platform +spec: + replicas: 1 + selector: + matchLabels: + app: database + template: + metadata: + labels: + app: database + spec: + containers: + - name: database + image: postgres:17-alpine + ports: + - containerPort: 5432 + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + env: + - name: POSTGRES_USER + valueFrom: + configMapKeyRef: + name: platform-config + key: SPRING_DATASOURCE_USERNAME + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: platform-secrets + key: POSTGRES_PASSWORD + - name: POSTGRES_DB + value: platform diff --git a/k8s/20-account.yaml b/k8s/20-account.yaml new file mode 100644 index 0000000..8cc37fc --- /dev/null +++ b/k8s/20-account.yaml @@ -0,0 +1,65 @@ +apiVersion: v1 +kind: Service +metadata: + name: account + namespace: platform +spec: + selector: + app: account + ports: + - port: 8080 + targetPort: 8080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: account + namespace: platform +spec: + replicas: 3 + selector: + matchLabels: + app: account + template: + metadata: + labels: + app: account + spec: + containers: + - name: account + image: microsserv52-account:local + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8080 + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + envFrom: + - configMapRef: + name: platform-config + - secretRef: + name: platform-secrets + env: + - name: SPRING_DATASOURCE_URL + valueFrom: + secretKeyRef: + name: platform-secrets + key: SPRING_DATASOURCE_URL + - name: SPRING_DATASOURCE_USERNAME + valueFrom: + secretKeyRef: + name: platform-secrets + key: SPRING_DATASOURCE_USERNAME + - name: SPRING_DATASOURCE_PASSWORD + valueFrom: + secretKeyRef: + name: platform-secrets + key: POSTGRES_PASSWORD + readinessProbe: + httpGet: + path: /actuator/health + port: 8080 diff --git a/k8s/30-auth.yaml b/k8s/30-auth.yaml new file mode 100644 index 0000000..44baf70 --- /dev/null +++ b/k8s/30-auth.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: Service +metadata: + name: auth + namespace: platform +spec: + selector: + app: auth + ports: + - port: 8080 + targetPort: 8080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: auth + namespace: platform +spec: + replicas: 3 + selector: + matchLabels: + app: auth + template: + metadata: + labels: + app: auth + spec: + containers: + - name: auth + image: microsserv52-auth:local + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8080 + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + envFrom: + - configMapRef: + name: platform-config + - secretRef: + name: platform-secrets + readinessProbe: + httpGet: + path: /actuator/health + port: 8080 diff --git a/k8s/40-order.yaml b/k8s/40-order.yaml new file mode 100644 index 0000000..4ff34e3 --- /dev/null +++ b/k8s/40-order.yaml @@ -0,0 +1,65 @@ +apiVersion: v1 +kind: Service +metadata: + name: order + namespace: platform +spec: + selector: + app: order + ports: + - port: 8080 + targetPort: 8080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: order + namespace: platform +spec: + replicas: 3 + selector: + matchLabels: + app: order + template: + metadata: + labels: + app: order + spec: + containers: + - name: order + image: microsserv52-order:local + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8080 + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + envFrom: + - configMapRef: + name: platform-config + - secretRef: + name: platform-secrets + env: + - name: SPRING_DATASOURCE_URL + valueFrom: + secretKeyRef: + name: platform-secrets + key: SPRING_DATASOURCE_URL + - name: SPRING_DATASOURCE_USERNAME + valueFrom: + secretKeyRef: + name: platform-secrets + key: SPRING_DATASOURCE_USERNAME + - name: SPRING_DATASOURCE_PASSWORD + valueFrom: + secretKeyRef: + name: platform-secrets + key: POSTGRES_PASSWORD + readinessProbe: + httpGet: + path: /actuator/health + port: 8080 diff --git a/k8s/50-gateway.yaml b/k8s/50-gateway.yaml new file mode 100644 index 0000000..1944024 --- /dev/null +++ b/k8s/50-gateway.yaml @@ -0,0 +1,49 @@ +apiVersion: v1 +kind: Service +metadata: + name: gateway + namespace: platform +spec: + selector: + app: gateway + ports: + - port: 8080 + targetPort: 8080 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gateway + namespace: platform +spec: + replicas: 3 + selector: + matchLabels: + app: gateway + template: + metadata: + labels: + app: gateway + spec: + containers: + - name: gateway + image: microsserv52-gateway:local + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8080 + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + envFrom: + - configMapRef: + name: platform-config + - secretRef: + name: platform-secrets + readinessProbe: + httpGet: + path: /actuator/health + port: 8080 diff --git a/k8s/60-ingress.yaml b/k8s/60-ingress.yaml new file mode 100644 index 0000000..6bcc55f --- /dev/null +++ b/k8s/60-ingress.yaml @@ -0,0 +1,19 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: platform + namespace: platform + annotations: + nginx.ingress.kubernetes.io/proxy-read-timeout: "60" +spec: + ingressClassName: nginx + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: gateway + port: + number: 8080 diff --git a/k8s/70-hpa.yaml b/k8s/70-hpa.yaml new file mode 100644 index 0000000..9719814 --- /dev/null +++ b/k8s/70-hpa.yaml @@ -0,0 +1,99 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: gateway + namespace: platform +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: gateway + minReplicas: 3 + maxReplicas: 10 + behavior: + scaleUp: + stabilizationWindowSeconds: 0 + policies: + - type: Percent + value: 100 + periodSeconds: 60 + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Percent + value: 25 + periodSeconds: 60 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 60 +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: order + namespace: platform +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: order + minReplicas: 3 + maxReplicas: 10 + behavior: + scaleDown: + stabilizationWindowSeconds: 300 + policies: + - type: Percent + value: 25 + periodSeconds: 60 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 60 +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: account + namespace: platform +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: account + minReplicas: 3 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 60 +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: auth + namespace: platform +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: auth + minReplicas: 3 + maxReplicas: 10 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 60 diff --git a/k8s/71-pdb.yaml b/k8s/71-pdb.yaml new file mode 100644 index 0000000..4d113d5 --- /dev/null +++ b/k8s/71-pdb.yaml @@ -0,0 +1,43 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: gateway + namespace: platform +spec: + minAvailable: 2 + selector: + matchLabels: + app: gateway +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: order + namespace: platform +spec: + minAvailable: 2 + selector: + matchLabels: + app: order +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: account + namespace: platform +spec: + minAvailable: 2 + selector: + matchLabels: + app: account +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: auth + namespace: platform +spec: + minAvailable: 2 + selector: + matchLabels: + app: auth diff --git a/k8s/README.md b/k8s/README.md index 40faa8d..a703320 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -1,5 +1,11 @@ # Kubernetes deployment +Kubernetes is the production backend. It does not use local Compose defaults: +database credentials, JWT credentials, and the production database URL must +come from a Kubernetes Secret or the deployment workflow. Product and Exchange +are external services and their production base URLs must be configured as +`PRODUCT_URL` and `EXCHANGE_URL`. + The manifest targets MicroK8s with the Nginx ingress addon and can be adapted to AWS EKS by replacing local image names with registry references. @@ -13,11 +19,81 @@ microk8s kubectl -n platform create secret generic platform-secrets \ --from-literal=POSTGRES_PASSWORD="$(openssl rand -base64 24)" \ --from-literal=SPRING_DATASOURCE_URL="jdbc:postgresql://database:5432/platform" \ --from-literal=SPRING_DATASOURCE_USERNAME="admin" -microk8s kubectl apply -f k8s/platform.yaml ``` -For EKS, configure the GitHub Actions secrets listed in `.env.example` and -run the manual `Deploy to AWS EKS` workflow. Set -`SPRING_DATASOURCE_URL` to an Amazon RDS PostgreSQL endpoint; the in-cluster -PostgreSQL deployment is intended for local MicroK8s only. Use an AWS Load -Balancer Controller or Nginx Ingress Controller for the `Ingress` resource. +For local MicroK8s, set `PRODUCT_URL` and `EXCHANGE_URL` in +`00-namespace-config.yaml` to reachable service URLs, then apply: + +```bash +microk8s start +microk8s status --wait-ready +microk8s enable dns metrics-server ingress +microk8s kubectl apply -f k8s/ +``` + +The HPA resources in `70-hpa.yaml` require the `metrics-server` addon. Verify +it before testing autoscaling: + +```bash +microk8s kubectl top nodes +microk8s kubectl top pods -n platform +microk8s kubectl get hpa -n platform +``` + +For a local HPA test, load the application images into MicroK8s or change the +manifest image references to registry images, deploy `70-hpa.yaml` and +`71-pdb.yaml`, then generate traffic with the k6 procedure in +[`load-testing.md`](../docs/operations/load-testing.md). HPA cannot scale a +Deployment whose pods are not running and ready. + +The Kubernetes resources are split by responsibility: + +- `00-namespace-config.yaml`: namespace and shared service configuration. +- `05-redis.yaml`: disposable local Redis. +- `10-database.yaml`: disposable local PostgreSQL Service and Deployment. +- `20-account.yaml`, `30-auth.yaml`, `40-order.yaml`, and `50-gateway.yaml`: each + microservice's Service and Deployment. +- `60-ingress.yaml`: Nginx Ingress for the gateway. +- `70-hpa.yaml`, `71-pdb.yaml`: autoscaling and voluntary-disruption limits. + +The `10-database.yaml` PostgreSQL deployment is intended only for local +MicroK8s. Do not apply it to EKS; EKS uses the production database URL in +`SPRING_DATASOURCE_URL`. + +For EKS, configure these GitHub Actions secrets: + +- `AWS_REGION` +- `AWS_DEPLOY_ROLE_ARN` +- `EKS_CLUSTER_NAME` +- `JWT_SECRET` +- `INTERNAL_SERVICE_SECRET` +- `SPRING_DATASOURCE_URL` +- `SPRING_DATASOURCE_USERNAME` +- `SPRING_DATASOURCE_PASSWORD` +- `CACHE_REDIS_URL` (may contain Redis credentials; stored in the runtime Secret) + +Also configure repository variables `PRODUCT_URL` and `EXCHANGE_URL`. Run the +manual `Deploy to AWS EKS` workflow. It deploys the application manifests, +skips the local PostgreSQL deployment, and patches the ConfigMap with the +external service URLs. Use an AWS Load Balancer Controller or Nginx Ingress +Controller for the `Ingress` resource. + +### Redis options + +Local MicroK8s includes the single-replica `05-redis.yaml` deployment and +advertises `redis://redis:6379` through `CACHE_REDIS_URL`. This is suitable for +development and cache experiments only; it has no high-availability or durable-storage guarantee. + +For EKS, set the `CACHE_REDIS_URL` repository secret to a reachable managed or +external Redis endpoint, such as Redis Cloud or Amazon ElastiCache. The EKS +workflow stores `CACHE_REDIS_URL` in `platform-secrets` with that value and does not deploy the +single-replica local Redis manifest. + +The current platform repository only wires the endpoint into service +configuration. Each service must implement and test cache-aside behavior, +TTL, invalidation and stampede protection before Redis provides a functional +performance mitigation. + +The Exchange provider URL and credentials are configuration for the external +Exchange service itself. Configure them in that service's deployment or secret +manager; they are not stored in this platform repository. diff --git a/k8s/platform.yaml b/k8s/platform.yaml deleted file mode 100644 index d3cfb69..0000000 --- a/k8s/platform.yaml +++ /dev/null @@ -1,286 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: platform ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: platform-config - namespace: platform -data: - SPRING_DATASOURCE_URL: jdbc:postgresql://database:5432/platform - SPRING_DATASOURCE_USERNAME: admin - ACCOUNT_URL: http://account:8080 - AUTH_URL: http://auth:8080 - ORDER_URL: http://order:8080 ---- -apiVersion: v1 -kind: Service -metadata: - name: database - namespace: platform -spec: - selector: - app: database - ports: - - port: 5432 - targetPort: 5432 ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: database - namespace: platform -spec: - replicas: 1 - selector: - matchLabels: - app: database - template: - metadata: - labels: - app: database - spec: - containers: - - name: database - image: postgres:17 - ports: - - containerPort: 5432 - env: - - name: POSTGRES_USER - valueFrom: - configMapKeyRef: - name: platform-config - key: SPRING_DATASOURCE_USERNAME - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: platform-secrets - key: POSTGRES_PASSWORD - - name: POSTGRES_DB - value: platform ---- -apiVersion: v1 -kind: Service -metadata: - name: account - namespace: platform -spec: - selector: - app: account - ports: - - port: 8080 - targetPort: 8080 ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: account - namespace: platform -spec: - replicas: 3 - selector: - matchLabels: - app: account - template: - metadata: - labels: - app: account - spec: - containers: - - name: account - image: Microsserv52-account:latest - imagePullPolicy: IfNotPresent - ports: - - containerPort: 8080 - envFrom: - - configMapRef: - name: platform-config - - secretRef: - name: platform-secrets - env: - - name: SPRING_DATASOURCE_URL - valueFrom: - secretKeyRef: - name: platform-secrets - key: SPRING_DATASOURCE_URL - - name: SPRING_DATASOURCE_USERNAME - valueFrom: - secretKeyRef: - name: platform-secrets - key: SPRING_DATASOURCE_USERNAME - - name: SPRING_DATASOURCE_PASSWORD - valueFrom: - secretKeyRef: - name: platform-secrets - key: POSTGRES_PASSWORD - readinessProbe: - httpGet: - path: /actuator/health - port: 8080 ---- -apiVersion: v1 -kind: Service -metadata: - name: auth - namespace: platform -spec: - selector: - app: auth - ports: - - port: 8080 - targetPort: 8080 ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: auth - namespace: platform -spec: - replicas: 3 - selector: - matchLabels: - app: auth - template: - metadata: - labels: - app: auth - spec: - containers: - - name: auth - image: Microsserv52-auth:latest - imagePullPolicy: IfNotPresent - ports: - - containerPort: 8080 - envFrom: - - configMapRef: - name: platform-config - - secretRef: - name: platform-secrets - readinessProbe: - httpGet: - path: /actuator/health - port: 8080 ---- -apiVersion: v1 -kind: Service -metadata: - name: order - namespace: platform -spec: - selector: - app: order - ports: - - port: 8080 - targetPort: 8080 ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: order - namespace: platform -spec: - replicas: 3 - selector: - matchLabels: - app: order - template: - metadata: - labels: - app: order - spec: - containers: - - name: order - image: Microsserv52-order:latest - imagePullPolicy: IfNotPresent - ports: - - containerPort: 8080 - envFrom: - - configMapRef: - name: platform-config - - secretRef: - name: platform-secrets - env: - - name: SPRING_DATASOURCE_URL - valueFrom: - secretKeyRef: - name: platform-secrets - key: SPRING_DATASOURCE_URL - - name: SPRING_DATASOURCE_USERNAME - valueFrom: - secretKeyRef: - name: platform-secrets - key: SPRING_DATASOURCE_USERNAME - - name: SPRING_DATASOURCE_PASSWORD - valueFrom: - secretKeyRef: - name: platform-secrets - key: POSTGRES_PASSWORD - readinessProbe: - httpGet: - path: /actuator/health - port: 8080 ---- -apiVersion: v1 -kind: Service -metadata: - name: gateway - namespace: platform -spec: - selector: - app: gateway - ports: - - port: 8080 - targetPort: 8080 ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: gateway - namespace: platform -spec: - replicas: 3 - selector: - matchLabels: - app: gateway - template: - metadata: - labels: - app: gateway - spec: - containers: - - name: gateway - image: Microsserv52-gateway:latest - imagePullPolicy: IfNotPresent - ports: - - containerPort: 8080 - envFrom: - - configMapRef: - name: platform-config - - secretRef: - name: platform-secrets - readinessProbe: - httpGet: - path: /actuator/health - port: 8080 ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: platform - namespace: platform - annotations: - nginx.ingress.kubernetes.io/proxy-read-timeout: "60" -spec: - ingressClassName: nginx - rules: - - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: gateway - port: - number: 8080 diff --git a/mocks/exchange-provider/Dockerfile b/mocks/exchange-provider/Dockerfile new file mode 100644 index 0000000..b974208 --- /dev/null +++ b/mocks/exchange-provider/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.13-alpine + +WORKDIR /app +COPY server.py . + +EXPOSE 8080 +CMD ["python", "server.py"] diff --git a/mocks/exchange-provider/server.py b/mocks/exchange-provider/server.py new file mode 100644 index 0000000..53c0fff --- /dev/null +++ b/mocks/exchange-provider/server.py @@ -0,0 +1,60 @@ +import json +from datetime import UTC, datetime +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import parse_qs, urlparse + +RATES = { + "EUR": 0.92, + "BRL": 5.10, + "GBP": 0.79, + "JPY": 150.0, +} + + +class ExchangeProviderHandler(BaseHTTPRequestHandler): + def do_GET(self): + request = urlparse(self.path) + if request.path == "/health": + self.send_json({"status": "ok"}) + return + + if request.path not in ("/latest", "/v1/latest", "/latest.json"): + self.send_error(404, "Not found") + return + + query = parse_qs(request.query) + base = query.get("base", ["USD"])[0].upper() + symbols = query.get("symbols", [None])[0] + requested = [item.upper() for item in symbols.split(",")] if symbols else None + rates = {**RATES, "USD": 1.0} + if base not in rates: + self.send_error(400, "Unsupported base currency") + return + base_rate = rates[base] + rates = {currency: rate / base_rate for currency, rate in rates.items()} + if requested: + if any(currency not in rates for currency in requested): + self.send_error(400, "Unsupported quote currency") + return + rates = { + currency: rates[currency] for currency in requested if currency in rates + } + + self.send_json( + {"base": base, "date": datetime.now(UTC).date().isoformat(), "rates": rates} + ) + + def send_json(self, payload): + body = json.dumps(payload).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format, *args): + return + + +if __name__ == "__main__": + ThreadingHTTPServer(("0.0.0.0", 8080), ExchangeProviderHandler).serve_forever() diff --git a/tests/infrastructure/test_exchange_provider.py b/tests/infrastructure/test_exchange_provider.py new file mode 100644 index 0000000..a46f171 --- /dev/null +++ b/tests/infrastructure/test_exchange_provider.py @@ -0,0 +1,56 @@ +import importlib.util +import json +import threading +import unittest +from http.server import ThreadingHTTPServer +from pathlib import Path +from urllib.error import HTTPError +from urllib.request import urlopen + +source = Path(__file__).resolve().parents[2] / "mocks/exchange-provider/server.py" +spec = importlib.util.spec_from_file_location("exchange_provider", source) +provider = importlib.util.module_from_spec(spec) +spec.loader.exec_module(provider) + + +class ExchangeProviderTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.server = ThreadingHTTPServer( + ("127.0.0.1", 0), provider.ExchangeProviderHandler + ) + cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True) + cls.thread.start() + cls.base_url = f"http://127.0.0.1:{cls.server.server_port}" + + @classmethod + def tearDownClass(cls): + cls.server.shutdown() + cls.server.server_close() + cls.thread.join() + + def get_json(self, path): + with urlopen(self.base_url + path, timeout=5) as response: + return json.load(response) + + def test_rebases_rates_and_filters_symbols(self): + result = self.get_json("/latest?base=EUR&symbols=USD,BRL,EUR") + self.assertEqual(result["base"], "EUR") + self.assertEqual(set(result["rates"]), {"USD", "BRL", "EUR"}) + self.assertAlmostEqual(result["rates"]["USD"], 1 / 0.92) + self.assertAlmostEqual(result["rates"]["BRL"], 5.10 / 0.92) + self.assertEqual(result["rates"]["EUR"], 1) + + def test_rejects_unsupported_currencies(self): + for path in ("/latest?base=XXX", "/latest?symbols=USD,XXX"): + with self.subTest(path=path), self.assertRaises(HTTPError) as raised: + self.get_json(path) + self.assertEqual(raised.exception.code, 400) + + def test_health_and_default_usd(self): + self.assertEqual(self.get_json("/health"), {"status": "ok"}) + self.assertEqual(self.get_json("/v1/latest")["rates"]["USD"], 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/k6/platform.js b/tests/k6/platform.js new file mode 100644 index 0000000..0310699 --- /dev/null +++ b/tests/k6/platform.js @@ -0,0 +1,38 @@ +import http from "k6/http"; +import { check, sleep } from "k6"; + +const baseUrl = (__ENV.BASE_URL || "http://localhost:8080").replace(/\/$/, ""); +const paths = (__ENV.PATHS || "/api/products") + .split(",") + .map((path) => path.trim()) + .filter(Boolean); + +http.setResponseCallback(http.expectedStatuses(200)); + +export const options = { + scenarios: { + steady: { + executor: "constant-arrival-rate", + rate: Number(__ENV.RATE || 10), + timeUnit: "1s", + duration: __ENV.DURATION || "2m", + preAllocatedVUs: Number(__ENV.PRE_ALLOCATED_VUS || 10), + maxVUs: Number(__ENV.MAX_VUS || 50), + }, + }, + thresholds: { + http_req_failed: ["rate<0.05"], + http_req_duration: ["p(95)<1000", "p(99)<2000"], + checks: ["rate>0.95"], + }, +}; + +export default function () { + for (const path of paths) { + const response = http.get(`${baseUrl}${path}`); + check(response, { + "request returns an expected API response": (res) => res.status === 200, + }); + } + sleep(0.1); +}