From 321f0b1dcd5acb708325f0feb3ab3f00c94319ae Mon Sep 17 00:00:00 2001 From: 0x48core Date: Fri, 9 Oct 2026 16:13:45 +0700 Subject: [PATCH] feat(lab-08): zero-downtime rolling deploys and rollback - deploy.sh: start the new container next to the old one, wait until healthy, then stop the old one gracefully; abort and keep the old version if the new one never becomes healthy; record deploy history - rollback.sh: redeploy the previous successful tag - hello-api: drain on SIGTERM (SHUTDOWN_DELAY) by failing /health before closing the listener; unit test for draining - Dockerfile: --start-interval=1s so new containers turn healthy fast - Traefik: retry middleware and a /health service check for hello-api - Lab 08 guide and knowledge base with measured results (74,746 failed requests before, 0 after) Co-Authored-By: Claude Opus 5.5 --- .gitignore | 3 + README.md | 2 +- apps/hello-api/Dockerfile | 4 +- apps/hello-api/main.go | 28 ++++ apps/hello-api/main_test.go | 14 ++ labs/08-zero-downtime-deploy/README.md | 178 +++++++++++++++++++++- labs/08-zero-downtime-deploy/knowledge.md | 121 +++++++++++++++ stacks/README.md | 2 +- stacks/hello-api/compose.traefik.yaml | 9 ++ stacks/hello-api/deploy.sh | 57 ++++++- stacks/hello-api/rollback.sh | 19 +++ stacks/traefik/dynamic/middlewares.yml | 8 + 12 files changed, 433 insertions(+), 12 deletions(-) create mode 100644 labs/08-zero-downtime-deploy/knowledge.md create mode 100755 stacks/hello-api/rollback.sh diff --git a/.gitignore b/.gitignore index b117146..b7caadc 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,6 @@ stacks/traefik/dynamic/tls.yml # Lab scratch space labs/**/scratch/ + +# Deploy history written by stacks/*/deploy.sh +.deploy-history diff --git a/README.md b/README.md index 0a47f8f..32ae450 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Status: ⬜ todo Β· 🟨 in progress Β· βœ… done | # | Lab | Status | |---|---|---| | 07 | [CI/CD pipeline](labs/07-cicd-pipeline/) | 🟨 | -| 08 | [Zero-downtime deployment](labs/08-zero-downtime-deploy/) | ⬜ | +| 08 | [Zero-downtime deployment](labs/08-zero-downtime-deploy/) | 🟨 | ### Phase 4 β€” Infrastructure as Code | # | Lab | Status | diff --git a/apps/hello-api/Dockerfile b/apps/hello-api/Dockerfile index 7bb3022..878a62b 100644 --- a/apps/hello-api/Dockerfile +++ b/apps/hello-api/Dockerfile @@ -29,7 +29,9 @@ USER nonroot:nonroot ENV HOST=0.0.0.0 PORT=8000 EXPOSE 8000 -HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ +# --start-interval: check every second while starting, so a new container is +# marked healthy (and gets traffic from Traefik) quickly during a rolling deploy. +HEALTHCHECK --interval=10s --timeout=3s --start-period=10s --start-interval=1s --retries=3 \ CMD ["/hello-api", "healthcheck"] # Exec form: hello-api is PID 1 and receives SIGTERM directly from docker stop. diff --git a/apps/hello-api/main.go b/apps/hello-api/main.go index a11ed2a..3909677 100644 --- a/apps/hello-api/main.go +++ b/apps/hello-api/main.go @@ -26,6 +26,8 @@ import ( "net/http" "os" "os/signal" + "strconv" + "sync/atomic" "syscall" "time" ) @@ -33,6 +35,10 @@ import ( // version is set at build time: go build -ldflags "-X main.version=1.0.0" var version = "dev" +// draining is set on SIGTERM: /health starts failing so load balancers stop +// sending new requests, while everything else keeps working (lab 08). +var draining atomic.Bool + func main() { if len(os.Args) > 1 && os.Args[1] == "healthcheck" { os.Exit(healthcheck()) @@ -65,6 +71,15 @@ func main() { }() <-ctx.Done() + + // Drain: fail /health and keep serving for SHUTDOWN_DELAY, so the proxy's + // health check removes this instance before it stops accepting connections. + if delay := shutdownDelay(); delay > 0 { + draining.Store(true) + log.Printf("draining for %s", delay) + time.Sleep(delay) + } + log.Print("shutting down") shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -77,6 +92,10 @@ func main() { func newMux(st *store) http.Handler { mux := http.NewServeMux() mux.HandleFunc("GET /health", func(w http.ResponseWriter, r *http.Request) { + if draining.Load() { + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"status": "draining"}) + return + } writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) }) mux.HandleFunc("GET /{$}", func(w http.ResponseWriter, r *http.Request) { @@ -167,6 +186,15 @@ func logRequests(next http.Handler) http.Handler { }) } +// shutdownDelay reads SHUTDOWN_DELAY in seconds (default 0: no draining). +func shutdownDelay() time.Duration { + n, err := strconv.Atoi(getenv("SHUTDOWN_DELAY", "0")) + if err != nil || n < 0 { + return 0 + } + return time.Duration(n) * time.Second +} + func getenv(key, fallback string) string { if v := os.Getenv(key); v != "" { return v diff --git a/apps/hello-api/main_test.go b/apps/hello-api/main_test.go index 753a7c6..0cc8cc6 100644 --- a/apps/hello-api/main_test.go +++ b/apps/hello-api/main_test.go @@ -68,3 +68,17 @@ func TestClientIP(t *testing.T) { t.Errorf("client_ip = %v, want 192.0.2.1 without X-Real-IP", got) } } + +func TestHealthWhileDraining(t *testing.T) { + draining.Store(true) + defer draining.Store(false) + + rec, body := do(t, "GET", "/health", nil) + if rec.Code != http.StatusServiceUnavailable || body["status"] != "draining" { + t.Errorf("got %d %v, want 503 draining", rec.Code, body) + } + // Real traffic is still served while draining. + if rec, _ := do(t, "GET", "/", nil); rec.Code != http.StatusOK { + t.Errorf("GET / while draining = %d, want 200", rec.Code) + } +} diff --git a/labs/08-zero-downtime-deploy/README.md b/labs/08-zero-downtime-deploy/README.md index e543e4c..4355a0e 100644 --- a/labs/08-zero-downtime-deploy/README.md +++ b/labs/08-zero-downtime-deploy/README.md @@ -1,25 +1,193 @@ # Lab 08 β€” Zero-downtime deployment - **Phase:** 3 β€” CI/CD -- **Status:** ⬜ todo +- **Status:** 🟨 in progress - **Started / finished:** β€” / β€” +- **Environment:** Docker Desktop on the Mac (Traefik from lab 05); the same scripts run on the VPS via CI (lab 07) + +> πŸ“– **Learn first:** read [knowledge.md](knowledge.md) before starting the steps below. ## Goal -Deploy without users noticing. +Deploy a new version while users keep getting answers, roll back with one command, and survive a broken release without an outage. ## Done when -- [ ] Blue-green or rolling deploy implemented -- [ ] Proven with a load test during deploy (e.g. `hey`/`k6`) showing zero failed requests -- [ ] One-command rollback to the previous version +- [ ] Measured the old deploy (lab 07) under load: how many requests failed? +- [ ] Rolling deploy under load with **zero** failed requests +- [ ] One-command rollback under load with zero failed requests +- [ ] A broken release fails the deploy, and the old version keeps serving +- [ ] Explained in "Lessons learned" why each piece is needed: healthcheck, drain delay, Traefik health check, retry + +## How it works + +``` + Traefik (routes only to healthy containers) + β”‚ β”‚ + 1. start new ──▢ [old v1] ◀── traffic [new v2] starting… (not routed) + 2. new healthy ─▢ [old v1] ◀── traffic ──▢ [new v2] + 3. SIGTERM old ─▢ [old v1] draining: /health = 503, still serving + Traefik health check removes it ──▢ [new v2] gets all traffic + 4. old exits ─▢ [new v2] +``` + +| Piece | Where | Why it's needed | +|---|---|---| +| **Run old and new side by side** | [deploy.sh](../../stacks/hello-api/deploy.sh) (`--scale app=2 --no-recreate`) | There's never a moment with zero containers | +| **Fast Docker healthcheck** | [Dockerfile](../../apps/hello-api/Dockerfile) (`--start-interval=1s`) | The new container is marked healthy in seconds; Traefik only routes to healthy containers | +| **Drain on SIGTERM** | [main.go](../../apps/hello-api/main.go) (`SHUTDOWN_DELAY=5` in [compose.traefik.yaml](../../stacks/hello-api/compose.traefik.yaml)) | `/health` fails first, so Traefik stops sending traffic *before* the app closes its port | +| **Traefik health check** | `compose.traefik.yaml` (`/health` every 2 s) | Notices the draining container and removes it from the load balancer | +| **Retry middleware** | [middlewares.yml](../../stacks/traefik/dynamic/middlewares.yml) | Safety net: a request that hits a closing connection is retried on the other container | +| **Abort if unhealthy** | `deploy.sh` (`wait_healthy`) | A broken release is removed, and the old one never stopped | +| **Deploy history** | `.deploy-history` (git-ignored) + [rollback.sh](../../stacks/hello-api/rollback.sh) | Rollback = deploy the previous tag, with the same zero-downtime path | + +## Measured while preparing this lab + +`hey -z 30s -c 10` against Traefik, on Docker Desktop: + +| Scenario | Requests | Failed | +|---|---|---| +| Lab 07 `deploy.sh` (recreate) | 231,757 | **74,746** (74,000 Γ— `404`, 746 Γ— `502`) | +| Rolling, no drain delay yet | 109,150 | 3 Γ— `502` | +| No deploy (control) | 267,689 | 0 | +| **Rolling deploy** v1 β†’ v2 | 333,726 | **0** | +| **Rollback** v2 β†’ v1 | 300,328 | **0** | +| **Broken release** (deploy aborted) | 320,391 | **0**, v1 kept serving | + +The `404`s happened while no container was healthy: Traefik removes the router entirely (the lab 05 behaviour). The `502`s came from the old container closing its port while Traefik still sent it requests, which the drain delay fixed. ## Steps +### 0. Setup + +```bash +brew install hey # HTTP load generator +docker network ls | grep proxy || docker network create proxy +(cd stacks/traefik && docker compose up -d) +``` + +Build two versions locally to deploy (or pull two real `sha-…` tags from GHCR after lab 07): + +```bash +for t in sha-v1test sha-v2test; do + docker build --build-arg VERSION=$t -t ghcr.io/0x48core/hello-api:$t apps/hello-api +done +``` + +The `hey` command used below sends 10 parallel requests for 30 seconds to Traefik, with the right `Host` header: + +```bash +hey -z 30s -c 10 -host api.opsforge.localhost https://127.0.0.1/ +``` + +### 1. Measure the problem first + +Save the lab 07 version of `deploy.sh` (commit `9e37407`) next to the compose files, and use it for one deploy under load: + +```bash +git show 9e37407:stacks/hello-api/deploy.sh > stacks/hello-api/deploy-recreate.sh && chmod +x stacks/hello-api/deploy-recreate.sh + +PULL=0 stacks/hello-api/deploy.sh sha-v1test # first deploy, nothing to replace +hey -z 30s -c 10 -host api.opsforge.localhost https://127.0.0.1/ > /tmp/before.txt & +sleep 8; PULL=0 stacks/hello-api/deploy-recreate.sh sha-v2test; wait +grep -A4 'Status code distribution' /tmp/before.txt +rm stacks/hello-api/deploy-recreate.sh +``` + +Write the numbers in "What broke". Which status codes did you get, and why? + +### 2. Rolling deploy under load + +Read [deploy.sh](../../stacks/hello-api/deploy.sh) first. Then: + +```bash +hey -z 30s -c 10 -host api.opsforge.localhost https://127.0.0.1/ > /tmp/rolling.txt & +sleep 6; PULL=0 stacks/hello-api/deploy.sh sha-v1test; wait +grep -A4 'Status code distribution' /tmp/rolling.txt +``` + +In a second terminal during the deploy, watch the containers overlap: + +```bash +watch -n 0.5 'docker ps --filter name=hello-api-app --format "{{.Names}} {{.Image}} {{.Status}}"' +``` + +You'll see two app containers at once, the new one going `health: starting` β†’ `healthy`, then the old one disappearing. Look at the old container's logs for `draining for 5s` β†’ `shutting down`. + +### 3. Rollback under load + +```bash +cat stacks/hello-api/.deploy-history +hey -z 30s -c 10 -host api.opsforge.localhost https://127.0.0.1/ > /tmp/rollback.txt & +sleep 6; PULL=0 stacks/hello-api/rollback.sh; wait +grep -A4 'Status code distribution' /tmp/rollback.txt +curl -sk -H 'Host: api.opsforge.localhost' https://127.0.0.1/ # which version now? +``` + +### 4. Deploy a broken release + +Make an image whose healthcheck always fails: + +```bash +printf 'FROM ghcr.io/0x48core/hello-api:sha-v2test\nHEALTHCHECK --interval=1s --retries=2 CMD ["/hello-api", "nope"]\n' \ + | docker build -t ghcr.io/0x48core/hello-api:sha-broken - +hey -z 30s -c 10 -host api.opsforge.localhost https://127.0.0.1/ > /tmp/broken.txt & +sleep 4; HEALTH_TIMEOUT=20 PULL=0 stacks/hello-api/deploy.sh sha-broken; echo "exit: $?"; wait +grep -A4 'Status code distribution' /tmp/broken.txt +tail -1 stacks/hello-api/.deploy-history # broken tag NOT recorded +``` + +In CI, that non-zero exit marks the `deploy` job red, so you'd see it immediately. + +### 5. Remove one piece at a time (optional, but the best way to learn) + +Repeat step 2 after each change, then undo it: + +| Remove | Expected effect | +|---|---| +| `SHUTDOWN_DELAY` in `compose.traefik.yaml` | A few `502`s when the old container stops | +| `retry@file` from the router | More of them | +| `--start-interval=1s` in the Dockerfile (rebuild) | The deploy takes ~10 s longer; still zero errors | +| The `sleep 3` in `deploy.sh` | Possible errors if Traefik hasn't picked up the new container yet | + +### 6. Clean up + +```bash +(cd stacks/hello-api && APP_VERSION=x docker compose -f compose.yaml -f compose.traefik.yaml -f compose.deploy.yaml down -v) +rm -f stacks/hello-api/.deploy-history +docker rmi ghcr.io/0x48core/hello-api:sha-v1test ghcr.io/0x48core/hello-api:sha-v2test ghcr.io/0x48core/hello-api:sha-broken +``` + +## On the VPS + +Nothing changes: CI's `deploy` job (lab 07) calls the same `deploy.sh`. To roll back on the server: + +```bash +ssh opsforge '~/opsforge/stacks/hello-api/rollback.sh' +``` + +or re-run an older successful `hello-api` workflow run's deploy job in GitHub Actions. + ## Code +- [stacks/hello-api/deploy.sh](../../stacks/hello-api/deploy.sh), [stacks/hello-api/rollback.sh](../../stacks/hello-api/rollback.sh) +- [apps/hello-api/main.go](../../apps/hello-api/main.go) (drain), [Dockerfile](../../apps/hello-api/Dockerfile) (`--start-interval`) +- [stacks/hello-api/compose.traefik.yaml](../../stacks/hello-api/compose.traefik.yaml), [stacks/traefik/dynamic/middlewares.yml](../../stacks/traefik/dynamic/middlewares.yml) + ## What broke +_Write down errors and fixes here as you go._ + +Found while preparing this lab: +- A first load-test run reported "0 errors" because zsh ran `$H` (a whole command in one variable) as a single program name, so `hey` never ran. **Always check the raw output of a test that "passes".** Fixed by using a shell function. +- Container names keep counting up (`hello-api-app-5`): each deploy scales to 2 and removes the old one. Harmless, but don't hard-code container names in scripts or monitors. + ## Lessons learned ## References + +- [Traefik: retry middleware](https://doc.traefik.io/traefik/middlewares/http/retry/) +- [Traefik: service health checks](https://doc.traefik.io/traefik/routing/services/#health-check) +- [Dockerfile HEALTHCHECK](https://docs.docker.com/reference/dockerfile/#healthcheck) +- [Kubernetes: container lifecycle hooks (preStop)](https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/) +- [hey](https://github.com/rakyll/hey) diff --git a/labs/08-zero-downtime-deploy/knowledge.md b/labs/08-zero-downtime-deploy/knowledge.md new file mode 100644 index 0000000..6e3b0c4 --- /dev/null +++ b/labs/08-zero-downtime-deploy/knowledge.md @@ -0,0 +1,121 @@ +# Lab 08 β€” Knowledge base + +Study this **before** the hands-on part in [README.md](README.md). Add your own notes as you learn. + +## Why this matters + +If every deploy causes errors, people deploy less often, changes pile up, releases get bigger and riskier, and deploys cause even more problems. **Zero-downtime deploys break that cycle**: shipping becomes boring and safe, so you can do it many times a day. In lab 07, a single deploy failed about a third of requests for a few seconds. Here you remove that, and you measure it. + +## Core concepts + +### Deployment strategies + +| Strategy | How | Downtime | Extra capacity | Rollback | +|---|---|---|---|---| +| **Recreate** | Stop old, start new | βœ… yes | None | Redeploy old | +| **Rolling** | Replace instances one by one, new before old | None | +1 instance during deploy | Roll again with the old version | +| **Blue-green** | Run a full new copy ("green"), switch all traffic at once | None | 2Γ— during deploy | **Instant**: switch back to blue | +| **Canary** | Send a small % of traffic to the new version, increase if healthy | None | Small | Shift traffic back | + +This lab implements **rolling** (one instance β†’ two β†’ one). Blue-green with Traefik would be two always-running services plus a **weighted service** in a dynamic file that you flip from `blue: 100` to `green: 100`. Canary is the same file with `blue: 90, green: 10`. + +### What has to be true for zero downtime + +1. **There's always at least one instance that can serve.** Start the new one before stopping the old. +2. **The proxy only sends traffic to instances that are ready.** Healthchecks, and Traefik ignoring containers that aren't healthy. +3. **An instance stops getting new traffic *before* it stops accepting it.** Drain, then shut down. +4. **In-flight requests finish.** Graceful shutdown (`srv.Shutdown`, lab 02). +5. **A bad version never takes traffic, and never removes the good one.** Health-gated deploys. + +Each piece in this lab covers one of these. Removing any of them brings errors back, which the guide's step 5 lets you see. + +### Why a drain delay is needed + +``` +without drain with drain (SHUTDOWN_DELAY=5) +t=0 SIGTERM β†’ app closes port t=0 SIGTERM β†’ /health returns 503, app still serves +t=0+ Traefik still routes here β†’ 502 tβ‰ˆ2 Traefik health check fails β†’ server removed +tβ‰ˆ1 Traefik notices, removes it t=5 app closes port, nobody is sending to it βœ… +``` + +The proxy learns about changes **asynchronously** (Docker events, health-check intervals). There's always a window where it still believes the old instance is fine. Draining makes that window harmless. Kubernetes has the same problem, and the common fix is a `preStop` hook that sleeps for a few seconds. + +### Liveness, readiness, and draining + +Lab 04 separated `/health` (liveness) from `/ready` (readiness). Draining adds one more state: "alive, but please stop sending me work". hello-api signals it by failing `/health` while draining. In Kubernetes (lab 12), you'd fail the **readiness** probe, so the pod leaves the Service endpoints without being restarted. + +### Retries: a safety net, not a fix + +- Traefik's `retry` middleware retries **only network errors** (connection refused or reset), never HTTP error responses such as a 500. +- Retries are safe here because a connection that failed before the request reached the app did no work. +- ⚠️ Retrying requests that might have *reached* the app is risky for non-idempotent operations such as `POST /visits`: the visit could be counted twice. Prefer to prevent errors (draining) rather than retry them away. + +### Health-gated deploys and automatic abort + +`deploy.sh` waits for the new container to be `healthy`. If it isn't within `HEALTH_TIMEOUT`, it removes the new container and exits 1. The old version never stopped, so a broken release causes **no outage**, and CI shows the deploy as failed. Only successful deploys are written to `.deploy-history`, so `rollback.sh` never "rolls back" to a broken tag. + +### Rollback + +- **Rollback = deploying a known-good previous artifact.** That's only possible because lab 07 builds immutable `sha-…` images: the old version still exists, unchanged, in the registry. +- **Roll forward vs roll back:** sometimes the fix is a quick new deploy. Rolling back is for "stop the bleeding now, investigate later". +- ⚠️ **Database migrations** are what make rollback hard. If v2 changes the schema in a way v1 can't read, rolling back the app breaks it. The usual rule is **expand β†’ migrate β†’ contract**: first add new columns or tables in a backward-compatible way, deploy code that uses them, and remove the old ones only in a later release. hello-api's `CREATE TABLE IF NOT EXISTS` is trivially compatible. + +### Measuring instead of believing + +"Zero downtime" is a claim, so you prove it with a **load test during the deploy**: +- `hey -z 30s -c 10` keeps 10 concurrent clients busy, and reports every status code and error. +- Run a **control** (no deploy) to know the baseline: some tools report errors at the very end of a timed run. +- Check that the test actually ran. A "perfect" result from a test that didn't run is the most dangerous kind (see "What broke" in the README). + +## Key terms + +| Term | Meaning | +|---|---| +| Rolling deploy | Replace instances gradually, starting new ones before stopping old ones | +| Blue-green | Two full environments; switch traffic between them at once | +| Canary | Send a small share of traffic to the new version first | +| Draining | Stop accepting new work but finish current work before exiting | +| Health-gated | The deploy continues only once the new version passes health checks | +| Idempotent | Safe to repeat: doing it twice has the same effect as once (GET yes, POST /visits no) | +| Expand/contract | Schema changes in backward-compatible steps, so rollback stays possible | + +## Commands to know + +| Command | What it does | +|---|---| +| `stacks/hello-api/deploy.sh ` | Rolling, health-gated deploy | +| `stacks/hello-api/rollback.sh` | Deploy the previous successful tag | +| `hey -z 30s -c 10 -host ` | Load test for 30 s with 10 concurrent clients | +| `docker inspect --format '{{.State.Health.Status}}' ` | `starting`, `healthy`, or `unhealthy` | +| `docker inspect --format '{{json .State.Health}}' ` | Healthcheck history with output | +| `docker compose up -d --no-recreate --scale app=2 app` | Add a second instance without touching the first | +| `docker stop --time 30 ` | SIGTERM, then SIGKILL after 30 s | + +## Before you start, can you answer these? + +1. Why did lab 07's deploy produce `404`s, not only `502`s? +2. What are the five conditions for zero downtime, and which file handles each one? +3. Why must the app fail `/health` *before* it stops listening? +4. Why is retrying `POST /visits` riskier than retrying `GET /`? +5. What makes a rollback impossible even when the old image still exists? + +## After the lab, check yourself + +1. How many requests failed with the old deploy, and with the new one? +2. What happened in the old container's logs between SIGTERM and exit? +3. Which piece, when removed, brought errors back? Why that one? +4. Why does `rollback.sh` never pick the broken tag from step 4? +5. How would you build blue-green with Traefik instead? What would rollback look like then? +6. What extra capacity does a rolling deploy need, and does it matter on a 4 GB VPS? + +## My notes + +_Your own explanations, analogies, and "aha" moments._ + +## Further reading + +- [Martin Fowler: Blue-green deployment](https://martinfowler.com/bliki/BlueGreenDeployment.html) +- [Martin Fowler: Canary release](https://martinfowler.com/bliki/CanaryRelease.html) +- [Traefik: weighted round robin (blue-green, canary)](https://doc.traefik.io/traefik/routing/services/#weighted-round-robin-service) +- [Kubernetes: rolling updates](https://kubernetes.io/docs/tutorials/kubernetes-basics/update/update-intro/) +- [Expand and contract pattern](https://martinfowler.com/bliki/ParallelChange.html) diff --git a/stacks/README.md b/stacks/README.md index b7f485f..501e8db 100644 --- a/stacks/README.md +++ b/stacks/README.md @@ -5,7 +5,7 @@ Docker Compose stacks, one folder per stack (`stacks//compose.yaml`, plus | Stack | What | Lab | |---|---|---| | [traefik](traefik/) | Reverse proxy and single entry point (ports 80/443) | 05 | -| [hello-api](hello-api/) | hello-api + PostgreSQL + Redis; `compose.traefik.yaml` puts it behind Traefik; `compose.deploy.yaml` + `deploy.sh` run the GHCR image on a server | 04, 05, 07 | +| [hello-api](hello-api/) | hello-api + PostgreSQL + Redis; `compose.traefik.yaml` puts it behind Traefik; `compose.deploy.yaml` + `deploy.sh` (zero-downtime rolling deploy) + `rollback.sh` run the GHCR image on a server | 04, 05, 07, 08 | | [whoami](whoami/) | Echoes request headers; for testing the proxy | 05 | | [gitea](gitea/) | Self-hosted Git server (`git.`), SSH on 2222 | 06 | | [uptime-kuma](uptime-kuma/) | Uptime monitoring (`status.`) | 06 | diff --git a/stacks/hello-api/compose.traefik.yaml b/stacks/hello-api/compose.traefik.yaml index 44f340e..4515480 100644 --- a/stacks/hello-api/compose.traefik.yaml +++ b/stacks/hello-api/compose.traefik.yaml @@ -3,6 +3,10 @@ services: app: ports: !reset [] # drop the 127.0.0.1:8000 publish from compose.yaml + environment: + # On SIGTERM, fail /health for 5 s before shutting down, so Traefik's + # health check (every 2 s) stops routing here first (lab 08). + SHUTDOWN_DELAY: "5" networks: - default # still reaches db and cache - proxy # reachable by Traefik @@ -11,6 +15,11 @@ services: - traefik.http.routers.hello-api.rule=Host(`api.opsforge.localhost`) - traefik.http.routers.hello-api.entrypoints=websecure - traefik.http.services.hello-api.loadbalancer.server.port=8000 + # Zero-downtime deploys (lab 08): + - traefik.http.routers.hello-api.middlewares=retry@file + - traefik.http.services.hello-api.loadbalancer.healthcheck.path=/health + - traefik.http.services.hello-api.loadbalancer.healthcheck.interval=2s + - traefik.http.services.hello-api.loadbalancer.healthcheck.timeout=1s networks: proxy: diff --git a/stacks/hello-api/deploy.sh b/stacks/hello-api/deploy.sh index 7e3389a..f8ad059 100755 --- a/stacks/hello-api/deploy.sh +++ b/stacks/hello-api/deploy.sh @@ -1,18 +1,41 @@ #!/usr/bin/env bash -# Deploy a hello-api image tag on this server (lab 07). +# Deploy a hello-api image tag on this server with zero downtime (labs 07–08). # CI runs this over SSH; you can also run it by hand. # # Usage: ./deploy.sh e.g. ./deploy.sh sha-1a2b3c4 # PULL=0 ./deploy.sh skip the pull (image already local, e.g. a rehearsal) # +# Rolling deploy: +# 1. start a container with the new image NEXT TO the running one +# 2. wait until it is healthy; Traefik only routes to healthy containers +# 3. stop the old one gracefully (SIGTERM, in-flight requests finish) +# If the new container never becomes healthy, it is removed and the old one +# keeps serving: a failed deploy causes no outage. +# # Needs: .env next to this file (DB credentials), the proxy network, and Traefik. set -euo pipefail TAG="${1:?usage: $0 }" cd "$(dirname "$0")" export APP_VERSION="$TAG" +HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-60}" +HISTORY=.deploy-history compose() { docker compose -f compose.yaml -f compose.traefik.yaml -f compose.deploy.yaml "$@"; } +image_of() { docker inspect --format '{{.Config.Image}}' "$1"; } + +wait_healthy() { + local id=$1 status + for ((i = 0; i < HEALTH_TIMEOUT; i++)); do + status=$(docker inspect --format '{{.State.Health.Status}}' "$id") + case "$status" in + healthy) return 0 ;; + unhealthy) return 1 ;; + esac + sleep 1 + done + return 1 +} [[ -f .env ]] || { echo "missing .env (copy .env.example and set a password)" >&2; exit 1; } @@ -21,8 +44,34 @@ if [[ "${PULL:-1}" == "1" ]]; then compose pull app fi -# --wait blocks until the containers are healthy, and fails if they never are. -compose up -d --no-build --wait --wait-timeout 60 +old=$(compose ps -q app) + +if [[ -z "$old" ]]; then + # First deploy: nothing to replace. + compose up -d --no-build --wait --wait-timeout "$HEALTH_TIMEOUT" +else + [[ $(wc -l <<<"$old") -eq 1 ]] || { echo "expected one running app container, found: $old" >&2; exit 1; } + echo "==> Old: $(image_of "$old")" + + # Scale to 2 without touching the existing container: the new one gets the new image. + compose up -d --no-build --no-recreate --scale app=2 app + new=$(compose ps -q app | grep -v "$old") + echo "==> New: $(image_of "$new"), waiting for it to be healthy..." + + if ! wait_healthy "$new"; then + echo "!!> New container is not healthy; removing it. The old version keeps serving." >&2 + docker logs --tail 20 "$new" >&2 || true + docker rm -f "$new" >/dev/null + exit 1 + fi + + # Give Traefik a moment to add the new container, then retire the old one. + sleep 3 + echo "==> Stopping old container (graceful, up to 30 s)" + docker stop --time 30 "$old" >/dev/null + docker rm "$old" >/dev/null +fi -echo "==> Running: $(docker inspect --format '{{.Config.Image}}' "$(compose ps -q app)")" +echo "$TAG" >> "$HISTORY" +echo "==> Running: $(image_of "$(compose ps -q app)")" echo "==> Deployed $TAG" diff --git a/stacks/hello-api/rollback.sh b/stacks/hello-api/rollback.sh new file mode 100755 index 0000000..b8d70bf --- /dev/null +++ b/stacks/hello-api/rollback.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Roll back hello-api to the previously deployed tag, with the same +# zero-downtime rolling deploy (lab 08). +# +# Usage: ./rollback.sh +# PULL=0 ./rollback.sh skip the pull (image already local) +set -euo pipefail +cd "$(dirname "$0")" + +HISTORY=.deploy-history +[[ -f "$HISTORY" ]] || { echo "no $HISTORY yet: nothing to roll back to" >&2; exit 1; } + +current=$(tail -n 1 "$HISTORY") +# The most recent entry that differs from the current tag. +previous=$(grep -vxF "$current" "$HISTORY" | tail -n 1 || true) +[[ -n "$previous" ]] || { echo "no earlier tag than $current in $HISTORY" >&2; exit 1; } + +echo "==> Rolling back: $current -> $previous" +exec ./deploy.sh "$previous" diff --git a/stacks/traefik/dynamic/middlewares.yml b/stacks/traefik/dynamic/middlewares.yml index e86236b..a78cdd0 100644 --- a/stacks/traefik/dynamic/middlewares.yml +++ b/stacks/traefik/dynamic/middlewares.yml @@ -5,3 +5,11 @@ http: dashboard-auth: basicAuth: usersFile: /etc/traefik/secrets/dashboard-users + + # Retry a request on another server if the connection fails, e.g. when it + # hits a container that is shutting down during a rolling deploy (lab 08). + # Only network errors are retried, never HTTP error responses. + retry: + retry: + attempts: 3 + initialInterval: 100ms