feat(lab-11): observability stack with metrics, logs, and alerts

.gitignore

@@ -50,6 +50,10 @@ stacks/traefik/secrets/*
 !stacks/traefik/secrets/.gitkeep
 stacks/traefik/dynamic/tls.yml
 
+# Alertmanager notification secrets (Telegram token, Discord webhook)
+stacks/observability/alertmanager/secrets/*
+!stacks/observability/alertmanager/secrets/.gitkeep
+
 # Lab scratch space
 labs/**/scratch/
 

README.md

@@ -48,7 +48,7 @@ Status: ⬜ todo · 🟨 in progress · ✅ done
 ### Phase 5 — Observability & Kubernetes
 | # | Lab | Status |
 |---|---|---|
-| 11 | [Observability stack](labs/11-observability/) | ⬜ |
+| 11 | [Observability stack](labs/11-observability/) | 🟨 |
 | 12 | [K3s, Helm & GitOps](labs/12-k3s-gitops/) | ⬜ |
 
 ### Beyond

apps/hello-api/README.md

@@ -9,6 +9,7 @@ A tiny JSON HTTP app used as the deploy target across OpsForge labs. Written in
 | `GET /ready` | readiness: status of Postgres and Redis; `503` if a configured one is unreachable |
 | `GET /visits` | visit count from Redis and total rows from Postgres |
 | `POST /visits` | record a visit in both (needs Postgres and Redis) |
+| `GET /metrics` | Prometheus metrics: `http_requests_total`, `http_request_duration_seconds`, `hello_api_visits_recorded_total`, Go runtime |
 
 It shuts down gracefully on `SIGTERM`, so in-flight requests finish when systemd or Docker stops it.
 
@@ -51,3 +52,5 @@ To run with Postgres and Redis, use the Compose stack in [stacks/hello-api](../.
 - [Lab 04](../../labs/04-dockerize-app/): multi-stage [Dockerfile](Dockerfile) (distroless, non-root, ~22 MB) and a Compose stack
 - [Lab 05](../../labs/05-multi-service-proxy/): behind Traefik at `api.opsforge.localhost`
 - [Lab 07](../../labs/07-cicd-pipeline/): unit tests ([main_test.go](main_test.go)), CI/CD to GHCR, deploy over SSH
+- [Lab 08](../../labs/08-zero-downtime-deploy/): drains on SIGTERM (`SHUTDOWN_DELAY`)
+- [Lab 11](../../labs/11-observability/): `/metrics` ([metrics.go](metrics.go)), scraped via Docker labels

apps/hello-api/go.mod

@@ -4,16 +4,23 @@ go 1.26.0
 
 require (
 	github.com/jackc/pgx/v5 v5.11.0
+	github.com/prometheus/client_golang v1.25.0
 	github.com/redis/go-redis/v9 v9.23.0
 )
 
 require (
+	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/prometheus/client_model v0.6.3 // indirect
+	github.com/prometheus/common v0.72.0 // indirect
+	github.com/prometheus/procfs v0.22.0 // indirect
 	go.uber.org/atomic v1.12.0 // indirect
-	golang.org/x/sync v0.17.0 // indirect
+	golang.org/x/sync v0.23.0 // indirect
 	golang.org/x/sys v0.48.0 // indirect
-	golang.org/x/text v0.29.0 // indirect
+	golang.org/x/text v0.42.0 // indirect
+	google.golang.org/protobuf v1.36.12 // indirect
 )

apps/hello-api/go.sum

@@ -1,10 +1,14 @@
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
 github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
 github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
 github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
 github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
@@ -13,9 +17,23 @@ github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg=
 github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
 github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
 github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
+github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
+github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
 github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
 github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.25.0 h1:cDtW1TcFsMi1RJc8khwCKB4VhzwV04aKqbYtNOXuiK0=
+github.com/prometheus/client_golang v1.25.0/go.mod h1:j2X/4/kaQHLPnmyEU8zjmkndo1T317VpCIQEml8LTKk=
+github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo=
+github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM=
+github.com/prometheus/common v0.72.0 h1:tAYsE+sPJxIncDAobm4H5aQjmox9ZxEIIqPbiffa8G4=
+github.com/prometheus/common v0.72.0/go.mod h1:77NWqAQ2tXT7BIK40qjJdw5Acrsrg1TlHAnsQi3i6mk=
+github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics=
+github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58=
 github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s=
 github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -27,13 +45,19 @@ github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
 github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
 go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE=
 go.uber.org/atomic v1.12.0/go.mod h1:I6c4cg+6HCxRjfjSsYtApoFILnpc0CGUdGkXVqbYVNk=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
+go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
 go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
 go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
-golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
-golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
+golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
 golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
 golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
-golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
-golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
+golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
+golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
+google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
+google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

apps/hello-api/main.go

@@ -13,6 +13,7 @@
 //	GET  /ready   readiness: Postgres and Redis (if configured) are reachable
 //	GET  /visits  visit counts from Redis and Postgres
 //	POST /visits  record a visit in both
+//	GET  /metrics Prometheus metrics
 //
 // Postgres and Redis are optional, enabled by DATABASE_URL and REDIS_URL.
 package main
@@ -125,12 +126,16 @@ func newMux(st *store) http.Handler {
 	})
 	mux.HandleFunc("POST /visits", func(w http.ResponseWriter, r *http.Request) {
 		v, err := st.recordVisit(r.Context(), clientIP(r))
+		if err == nil {
+			visitsRecorded.Inc()
+		}
 		writeResult(w, v, err)
 	})
+	mux.Handle("GET /metrics", metricsHandler())
 	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		writeJSON(w, http.StatusNotFound, map[string]string{"error": "not found"})
 	})
-	return mux
+	return instrument(mux)
 }
 
 // healthcheck lets the container check itself without curl or wget,

apps/hello-api/main_test.go

@@ -82,3 +82,27 @@ func TestHealthWhileDraining(t *testing.T) {
 		t.Errorf("GET / while draining = %d, want 200", rec.Code)
 	}
 }
+
+func TestMetrics(t *testing.T) {
+	do(t, "GET", "/health", nil)
+	do(t, "GET", "/no/such/page", nil)
+
+	req := httptest.NewRequest("GET", "/metrics", nil)
+	rec := httptest.NewRecorder()
+	newMux(&store{}).ServeHTTP(rec, req)
+	if rec.Code != http.StatusOK {
+		t.Fatalf("GET /metrics = %d, want 200", rec.Code)
+	}
+	body := rec.Body.String()
+	for _, want := range []string{
+		`http_requests_total{code="200",method="GET",route="GET /health"}`,
+		`http_requests_total{code="404",method="GET",route="unmatched"}`, // raw path never becomes a label
+		`http_request_duration_seconds_bucket{method="GET",route="GET /health"`,
+		`hello_api_visits_recorded_total 0`,
+		`go_goroutines`,
+	} {
+		if !strings.Contains(body, want) {
+			t.Errorf("/metrics is missing %s", want)
+		}
+	}
+}

apps/hello-api/metrics.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+// Prometheus metrics (lab 11). Scraped from GET /metrics.
+var (
+	registry = prometheus.NewRegistry()
+
+	// RED metrics: Rate and Errors (by code) and Duration, per route.
+	httpRequests = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "http_requests_total",
+		Help: "HTTP requests handled, by method, route pattern, and status code.",
+	}, []string{"method", "route", "code"})
+
+	httpDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "http_request_duration_seconds",
+		Help:    "HTTP request latency, by method and route pattern.",
+		Buckets: []float64{.001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5},
+	}, []string{"method", "route"})
+
+	// A business metric: what the app is for, not just how it runs.
+	visitsRecorded = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "hello_api_visits_recorded_total",
+		Help: "Visits successfully recorded in Postgres and Redis.",
+	})
+)
+
+func init() {
+	registry.MustRegister(
+		httpRequests, httpDuration, visitsRecorded,
+		collectors.NewGoCollector(),                                       // goroutines, GC, memory
+		collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), // CPU, open files, RSS
+	)
+}
+
+func metricsHandler() http.Handler {
+	return promhttp.HandlerFor(registry, promhttp.HandlerOpts{})
+}
+
+// statusRecorder remembers the status code a handler wrote.
+type statusRecorder struct {
+	http.ResponseWriter
+	code int
+}
+
+func (r *statusRecorder) WriteHeader(code int) {
+	r.code = code
+	r.ResponseWriter.WriteHeader(code)
+}
+
+// instrument records every request. It uses the matched route pattern
+// (e.g. "GET /visits"), never the raw path, so that random URLs can't create
+// unlimited label values (high cardinality) in Prometheus.
+func instrument(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		start := time.Now()
+		rec := &statusRecorder{ResponseWriter: w, code: http.StatusOK}
+		next.ServeHTTP(rec, r)
+
+		route := r.Pattern // set by ServeMux on this request
+		if route == "" || route == "/" {
+			route = "unmatched"
+		}
+		httpRequests.WithLabelValues(r.Method, route, strconv.Itoa(rec.code)).Inc()
+		httpDuration.WithLabelValues(r.Method, route).Observe(time.Since(start).Seconds())
+	})
+}