From cc977f265c6e1b8d26ab4bd324ca53d1c6cba96d Mon Sep 17 00:00:00 2001 From: 0x48core Date: Fri, 9 Oct 2026 17:20:35 +0700 Subject: [PATCH] feat(lab-11): observability stack with metrics, logs, and alerts - stacks/observability: Prometheus, Alertmanager, Grafana, Loki, Alloy, node-exporter, cAdvisor; Docker API only through a read-only socket proxy; Grafana data sources and dashboard provisioned - hello-api: /metrics with RED metrics per route pattern (no high-cardinality paths) and a visits counter; tested - Traefik: Prometheus metrics entry point; scrape labels on Traefik and hello-api for Docker service discovery - Alert rules incl. ServiceMissing via absent(): discovered targets vanish instead of going down - Lab 11 guide and knowledge base with measured results Co-Authored-By: Claude Opus 5.5 --- .gitignore | 4 + README.md | 2 +- apps/hello-api/README.md | 3 + apps/hello-api/go.mod | 11 +- apps/hello-api/go.sum | 32 +- apps/hello-api/main.go | 7 +- apps/hello-api/main_test.go | 24 + apps/hello-api/metrics.go | 75 ++ labs/11-observability/README.md | 206 +++++- labs/11-observability/knowledge.md | 162 ++++ stacks/README.md | 1 + stacks/hello-api/compose.yaml | 5 + stacks/observability/.env.example | 3 + .../alertmanager/alertmanager.yml | 27 + .../alertmanager/secrets/.gitkeep | 0 stacks/observability/alloy/config.alloy | 40 + stacks/observability/compose.yaml | 163 +++++ .../grafana/dashboards/opsforge-overview.json | 690 ++++++++++++++++++ .../provisioning/dashboards/dashboards.yml | 9 + .../provisioning/datasources/datasources.yml | 15 + stacks/observability/loki/loki.yml | 37 + stacks/observability/prometheus/alerts.yml | 61 ++ .../observability/prometheus/prometheus.yml | 51 ++ stacks/traefik/compose.yaml | 4 + stacks/traefik/traefik.yml | 9 + 25 files changed, 1627 insertions(+), 14 deletions(-) create mode 100644 apps/hello-api/metrics.go create mode 100644 labs/11-observability/knowledge.md create mode 100644 stacks/observability/.env.example create mode 100644 stacks/observability/alertmanager/alertmanager.yml create mode 100644 stacks/observability/alertmanager/secrets/.gitkeep create mode 100644 stacks/observability/alloy/config.alloy create mode 100644 stacks/observability/compose.yaml create mode 100644 stacks/observability/grafana/dashboards/opsforge-overview.json create mode 100644 stacks/observability/grafana/provisioning/dashboards/dashboards.yml create mode 100644 stacks/observability/grafana/provisioning/datasources/datasources.yml create mode 100644 stacks/observability/loki/loki.yml create mode 100644 stacks/observability/prometheus/alerts.yml create mode 100644 stacks/observability/prometheus/prometheus.yml diff --git a/.gitignore b/.gitignore index 1a0231c..54a2493 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,10 @@ stacks/traefik/secrets/* !stacks/traefik/secrets/.gitkeep stacks/traefik/dynamic/tls.yml +# Alertmanager notification secrets (Telegram token, Discord webhook) +stacks/observability/alertmanager/secrets/* +!stacks/observability/alertmanager/secrets/.gitkeep + # Lab scratch space labs/**/scratch/ diff --git a/README.md b/README.md index a7bd139..0a1efb5 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Status: ⬜ todo Β· 🟨 in progress Β· βœ… done ### Phase 5 β€” Observability & Kubernetes | # | Lab | Status | |---|---|---| -| 11 | [Observability stack](labs/11-observability/) | ⬜ | +| 11 | [Observability stack](labs/11-observability/) | 🟨 | | 12 | [K3s, Helm & GitOps](labs/12-k3s-gitops/) | ⬜ | ### Beyond diff --git a/apps/hello-api/README.md b/apps/hello-api/README.md index dcfe1ae..d1aa7c2 100644 --- a/apps/hello-api/README.md +++ b/apps/hello-api/README.md @@ -9,6 +9,7 @@ A tiny JSON HTTP app used as the deploy target across OpsForge labs. Written in | `GET /ready` | readiness: status of Postgres and Redis; `503` if a configured one is unreachable | | `GET /visits` | visit count from Redis and total rows from Postgres | | `POST /visits` | record a visit in both (needs Postgres and Redis) | +| `GET /metrics` | Prometheus metrics: `http_requests_total`, `http_request_duration_seconds`, `hello_api_visits_recorded_total`, Go runtime | It shuts down gracefully on `SIGTERM`, so in-flight requests finish when systemd or Docker stops it. @@ -51,3 +52,5 @@ To run with Postgres and Redis, use the Compose stack in [stacks/hello-api](../. - [Lab 04](../../labs/04-dockerize-app/): multi-stage [Dockerfile](Dockerfile) (distroless, non-root, ~22 MB) and a Compose stack - [Lab 05](../../labs/05-multi-service-proxy/): behind Traefik at `api.opsforge.localhost` - [Lab 07](../../labs/07-cicd-pipeline/): unit tests ([main_test.go](main_test.go)), CI/CD to GHCR, deploy over SSH +- [Lab 08](../../labs/08-zero-downtime-deploy/): drains on SIGTERM (`SHUTDOWN_DELAY`) +- [Lab 11](../../labs/11-observability/): `/metrics` ([metrics.go](metrics.go)), scraped via Docker labels diff --git a/apps/hello-api/go.mod b/apps/hello-api/go.mod index 22540df..216853c 100644 --- a/apps/hello-api/go.mod +++ b/apps/hello-api/go.mod @@ -4,16 +4,23 @@ go 1.26.0 require ( github.com/jackc/pgx/v5 v5.11.0 + github.com/prometheus/client_golang v1.25.0 github.com/redis/go-redis/v9 v9.23.0 ) require ( + github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.3 // indirect + github.com/prometheus/common v0.72.0 // indirect + github.com/prometheus/procfs v0.22.0 // indirect go.uber.org/atomic v1.12.0 // indirect - golang.org/x/sync v0.17.0 // indirect + golang.org/x/sync v0.23.0 // indirect golang.org/x/sys v0.48.0 // indirect - golang.org/x/text v0.29.0 // indirect + golang.org/x/text v0.42.0 // indirect + google.golang.org/protobuf v1.36.12 // indirect ) diff --git a/apps/hello-api/go.sum b/apps/hello-api/go.sum index b00e9a2..1c84094 100644 --- a/apps/hello-api/go.sum +++ b/apps/hello-api/go.sum @@ -1,3 +1,5 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= @@ -5,6 +7,8 @@ github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= @@ -13,9 +17,23 @@ github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= +github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.25.0 h1:cDtW1TcFsMi1RJc8khwCKB4VhzwV04aKqbYtNOXuiK0= +github.com/prometheus/client_golang v1.25.0/go.mod h1:j2X/4/kaQHLPnmyEU8zjmkndo1T317VpCIQEml8LTKk= +github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo= +github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM= +github.com/prometheus/common v0.72.0 h1:tAYsE+sPJxIncDAobm4H5aQjmox9ZxEIIqPbiffa8G4= +github.com/prometheus/common v0.72.0/go.mod h1:77NWqAQ2tXT7BIK40qjJdw5Acrsrg1TlHAnsQi3i6mk= +github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics= +github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58= github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s= github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -27,13 +45,19 @@ github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE= go.uber.org/atomic v1.12.0/go.mod h1:I6c4cg+6HCxRjfjSsYtApoFILnpc0CGUdGkXVqbYVNk= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= -golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= -golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/apps/hello-api/main.go b/apps/hello-api/main.go index 3909677..8d53b50 100644 --- a/apps/hello-api/main.go +++ b/apps/hello-api/main.go @@ -13,6 +13,7 @@ // GET /ready readiness: Postgres and Redis (if configured) are reachable // GET /visits visit counts from Redis and Postgres // POST /visits record a visit in both +// GET /metrics Prometheus metrics // // Postgres and Redis are optional, enabled by DATABASE_URL and REDIS_URL. package main @@ -125,12 +126,16 @@ func newMux(st *store) http.Handler { }) mux.HandleFunc("POST /visits", func(w http.ResponseWriter, r *http.Request) { v, err := st.recordVisit(r.Context(), clientIP(r)) + if err == nil { + visitsRecorded.Inc() + } writeResult(w, v, err) }) + mux.Handle("GET /metrics", metricsHandler()) mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusNotFound, map[string]string{"error": "not found"}) }) - return mux + return instrument(mux) } // healthcheck lets the container check itself without curl or wget, diff --git a/apps/hello-api/main_test.go b/apps/hello-api/main_test.go index 0cc8cc6..ae6722c 100644 --- a/apps/hello-api/main_test.go +++ b/apps/hello-api/main_test.go @@ -82,3 +82,27 @@ func TestHealthWhileDraining(t *testing.T) { t.Errorf("GET / while draining = %d, want 200", rec.Code) } } + +func TestMetrics(t *testing.T) { + do(t, "GET", "/health", nil) + do(t, "GET", "/no/such/page", nil) + + req := httptest.NewRequest("GET", "/metrics", nil) + rec := httptest.NewRecorder() + newMux(&store{}).ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("GET /metrics = %d, want 200", rec.Code) + } + body := rec.Body.String() + for _, want := range []string{ + `http_requests_total{code="200",method="GET",route="GET /health"}`, + `http_requests_total{code="404",method="GET",route="unmatched"}`, // raw path never becomes a label + `http_request_duration_seconds_bucket{method="GET",route="GET /health"`, + `hello_api_visits_recorded_total 0`, + `go_goroutines`, + } { + if !strings.Contains(body, want) { + t.Errorf("/metrics is missing %s", want) + } + } +} diff --git a/apps/hello-api/metrics.go b/apps/hello-api/metrics.go new file mode 100644 index 0000000..0c4ead0 --- /dev/null +++ b/apps/hello-api/metrics.go @@ -0,0 +1,75 @@ +package main + +import ( + "net/http" + "strconv" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +// Prometheus metrics (lab 11). Scraped from GET /metrics. +var ( + registry = prometheus.NewRegistry() + + // RED metrics: Rate and Errors (by code) and Duration, per route. + httpRequests = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "http_requests_total", + Help: "HTTP requests handled, by method, route pattern, and status code.", + }, []string{"method", "route", "code"}) + + httpDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "http_request_duration_seconds", + Help: "HTTP request latency, by method and route pattern.", + Buckets: []float64{.001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5}, + }, []string{"method", "route"}) + + // A business metric: what the app is for, not just how it runs. + visitsRecorded = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "hello_api_visits_recorded_total", + Help: "Visits successfully recorded in Postgres and Redis.", + }) +) + +func init() { + registry.MustRegister( + httpRequests, httpDuration, visitsRecorded, + collectors.NewGoCollector(), // goroutines, GC, memory + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), // CPU, open files, RSS + ) +} + +func metricsHandler() http.Handler { + return promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) +} + +// statusRecorder remembers the status code a handler wrote. +type statusRecorder struct { + http.ResponseWriter + code int +} + +func (r *statusRecorder) WriteHeader(code int) { + r.code = code + r.ResponseWriter.WriteHeader(code) +} + +// instrument records every request. It uses the matched route pattern +// (e.g. "GET /visits"), never the raw path, so that random URLs can't create +// unlimited label values (high cardinality) in Prometheus. +func instrument(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + rec := &statusRecorder{ResponseWriter: w, code: http.StatusOK} + next.ServeHTTP(rec, r) + + route := r.Pattern // set by ServeMux on this request + if route == "" || route == "/" { + route = "unmatched" + } + httpRequests.WithLabelValues(r.Method, route, strconv.Itoa(rec.code)).Inc() + httpDuration.WithLabelValues(r.Method, route).Observe(time.Since(start).Seconds()) + }) +} diff --git a/labs/11-observability/README.md b/labs/11-observability/README.md index 4051ed1..42eb4a0 100644 --- a/labs/11-observability/README.md +++ b/labs/11-observability/README.md @@ -1,26 +1,220 @@ # Lab 11 β€” Observability stack - **Phase:** 5 β€” Observability & Kubernetes -- **Status:** ⬜ todo +- **Status:** 🟨 in progress - **Started / finished:** β€” / β€” +- **Environment:** Docker Desktop on the Mac, behind the Traefik stack from lab 05 + +> πŸ“– **Learn first:** read [knowledge.md](knowledge.md) before starting the steps below. ## Goal -See what the server is doing and get told when something is wrong. +See what the server and every service are doing (metrics, logs, dashboards), and be told when something is wrong (alerts), without logging in and running `docker ps`. ## Done when -- [ ] Prometheus + Node Exporter collecting CPU, RAM, disk -- [ ] Grafana dashboards for host and containers -- [ ] Loki collecting logs from all containers -- [ ] Alertmanager sends alerts to Telegram or Discord (tested by triggering one) +- [ ] Prometheus scrapes host (node-exporter), containers (cAdvisor), Traefik, and hello-api; all targets **up** +- [ ] hello-api exposes `/metrics`, and Prometheus finds it through Docker labels +- [ ] Grafana shows the provisioned **OpsForge overview** dashboard with live data +- [ ] Loki has the logs of every container; you found a hello-api error in Grafana β†’ Explore +- [ ] An alert went **pending β†’ firing β†’ resolved**, and you saw it in Alertmanager +- [ ] Optional: the alert arrived on Telegram or Discord +- [ ] RAM used by the whole stack noted in "Lessons learned" + +## Architecture + +``` + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ proxy network ────────────┐ +Browser ─HTTPS─▢ Traefik ┼─▢ grafana. ──▢ Grafana ─┬─▢ Prometheus ◀─ scrape ─┬─ Traefik :8082 + β”œβ”€β–Ά prometheus. (basic auth)β”‚ β”œβ”€ hello-api :8000/metrics (Docker labels) + └─▢ alertmanager. (auth) └─▢ Loki ◀── push ── Alloy β”‚ + β”‚ β”œβ”€ node-exporter (host) + observability network: β”‚ └─ cAdvisor (containers) + socket-proxy (read-only Docker API) ◀─────┴─ Prometheus, Alloy, cAdvisor + Prometheus ── alerts ──▢ Alertmanager ──▢ (Telegram / Discord) +``` + +| Service | Role | Image | +|---|---|---| +| Prometheus | Scrapes and stores metrics, evaluates alert rules | `prom/prometheus:v3.15.0` | +| Alertmanager | Groups and routes alerts to notifications | `prom/alertmanager:v0.34.1` | +| node-exporter | Host CPU, RAM, disk, network | `prom/node-exporter:v1.12.1` | +| cAdvisor | Per-container CPU and RAM | `ghcr.io/google/cadvisor:v0.60.6` | +| Loki | Stores logs, indexed only by labels | `grafana/loki:3.7.8` | +| Alloy | Reads container logs, pushes them to Loki | `grafana/alloy:v1.20.1` | +| socket-proxy | Read-only Docker API (no socket mounted into Prometheus or Alloy) | `tecnativa/docker-socket-proxy:v0.5.0` | +| Grafana | Dashboards and log exploration, provisioned from files | `grafana/grafana:13.2.3` | + +Files: [stacks/observability/](../../stacks/observability/). App metrics: [apps/hello-api/metrics.go](../../apps/hello-api/metrics.go). + +## Measured while preparing this lab + +| Check | Result | +|---|---| +| Config validation | promtool (config + **6 rules**), amtool, `loki -verify-config`, `alloy fmt`: all pass | +| Targets | prometheus, node, cadvisor, traefik, hello-api: all **up** | +| Dashboard | provisioned (18 panels); every panel query returns data (5xx: once errors happen) | +| Logs | hello-api lines in Loki, labels `project`, `service`, `container` | +| Alert test | stopped hello-api β†’ `ServiceMissing` pending β†’ firing (90 s) β†’ in Alertmanager β†’ resolved after restart | +| RAM | ~780 MiB for the whole stack (Alloy ~210, Loki ~205, cAdvisor ~120, Grafana ~110, Prometheus ~80) | ## Steps +### 0. Prerequisites + +```bash +docker network ls | grep proxy || docker network create proxy +cat stacks/traefik/secrets/dashboard-users # must contain admin:$2y$…; if empty: lab 05 step 2 +(cd stacks/traefik && docker compose up -d) +(cd stacks/hello-api && docker compose -f compose.yaml -f compose.traefik.yaml up -d --build --wait) +``` + +hello-api is rebuilt because it now has a `/metrics` endpoint. Look at it: + +```bash +curl -sk -H 'Host: api.opsforge.localhost' https://127.0.0.1/ >/dev/null +# hello-api is distroless (no curl inside), so look from a throwaway container on the proxy network: +docker run --rm --network proxy curlimages/curl:8.11.1 -s http://hello-api-app-1:8000/metrics | grep -E '^(http_requests_total|hello_api)' +``` + +### 1. Start the stack + +```bash +cd stacks/observability +cp .env.example .env # set GRAFANA_ADMIN_PASSWORD +docker compose up -d +docker compose ps +``` + +### 2. Prometheus: targets and queries + +Open `https://prometheus.opsforge.localhost` (dashboard login from lab 05). +- **Status β†’ Targets:** all up? Note how `traefik` and `hello-api` were found: **Docker labels** (`prometheus.scrape=true`) through the socket proxy. +- **Alerts:** the 6 rules from [alerts.yml](../../stacks/observability/prometheus/alerts.yml), all inactive. +- In **Query**, try these, one at a time: + +```promql +up +rate(http_requests_total{job="hello-api"}[1m]) +sum by (route, code) (http_requests_total{job="hello-api"}) +histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket{job="hello-api"}[5m]))) +sum by (service, code) (rate(traefik_service_requests_total[1m])) +topk(5, container_memory_working_set_bytes{name!=""}) +1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes +``` + +Generate traffic in another terminal and watch the numbers move: + +```bash +while true; do curl -sk -o /dev/null -X POST -H 'Host: api.opsforge.localhost' https://127.0.0.1/visits; sleep 0.2; done +``` + +### 3. Grafana + +Open `https://grafana.opsforge.localhost` and log in as `admin` with the password from `.env`. +- **Dashboards β†’ OpsForge β†’ OpsForge overview.** Find each panel's query: **Edit** β†’ see the PromQL. +- **Connections β†’ Data sources:** Prometheus and Loki are there, read-only. They came from [provisioning files](../../stacks/observability/grafana/provisioning/), not clicks, so a rebuilt server gets the same Grafana. +- Optional: **Dashboards β†’ New β†’ Import β†’ ID `1860`** (Node Exporter Full), with Prometheus as the data source. + +### 4. Logs in Loki + +**Explore β†’ Loki**, then try: + +```logql +{project="hello-api", service="app"} +{project="hello-api", service="app"} |= "POST /visits" +{project="observability"} |~ "(?i)error" +sum by (service) (count_over_time({project="hello-api"}[1m])) +``` + +Notice that ~90% of hello-api's log lines are `GET /health` (from Docker, Traefik, and Prometheus). That's the log noise you noted in lab 04. Filter it out: `{service="app"} != "/health"`. + +### 5. Break something, get an alert + +Make hello-api fail with 5xx errors: + +```bash +(cd ../hello-api && docker compose -f compose.yaml -f compose.traefik.yaml stop db) +for i in $(seq 1 20); do curl -sk -o /dev/null -w '%{http_code} ' -X POST -H 'Host: api.opsforge.localhost' https://127.0.0.1/visits; done +``` + +Find the errors in three places: the **5xx** dashboard panel, `{service="app"} |= "failed"` in Loki, and `http_requests_total{code="500"}` in Prometheus. Then start the DB again. + +Now stop the app entirely: + +```bash +(cd ../hello-api && docker compose -f compose.yaml -f compose.traefik.yaml stop app) +``` + +In Prometheus **Alerts**, watch `ServiceMissing` go **pending** (the `for: 1m` window) and then **firing**. Open `https://alertmanager.opsforge.localhost` and see it there. Then start the app again and watch it **resolve**. + +Why `ServiceMissing` and not `TargetDown`? Check `up{job="hello-api"}` while the app is stopped. See "What broke". + +### 6. Optional: notifications to Telegram or Discord + +1. Telegram: create a bot with **@BotFather**, then `echo '' > alertmanager/secrets/telegram-token`, and get your chat id from **@userinfobot**. + Discord: **Server settings β†’ Integrations β†’ Webhooks**, then `echo '' > alertmanager/secrets/discord-webhook`. +2. In [alertmanager.yml](../../stacks/observability/alertmanager/alertmanager.yml), uncomment that receiver and set `route.receiver` to its name. +3. Check and apply it: + +```bash +docker compose exec alertmanager amtool check-config /etc/alertmanager/alertmanager.yml +docker compose restart alertmanager +``` + +Repeat step 5. The message arrives after `group_wait` (30 s) once the alert fires, and again when it resolves. + +### 7. Look at the cost + +```bash +docker stats --no-stream --format '{{.Name}} {{.MemUsage}}' | grep observability +``` + +Add it up, and compare with a 4 GB VPS that also runs Traefik, hello-api, Gitea, and (lab 12) K3s. + +### 8. Clean up + +```bash +docker compose down # keeps data; add -v to delete metrics, logs, and Grafana state +``` + +## On the VPS + +The same stack works there. Two differences: +- node-exporter measures the **real** server instead of Docker Desktop's VM. You can use `/:/host:ro,rslave`. +- `DOMAIN` comes from `.env`, so you get `grafana.` with a Let's Encrypt certificate. + +Adding it to the Ansible `opsforge` role (copy the stack, write `.env` from the vault, `docker compose up`) is a good exercise. + ## Code +- [stacks/observability/](../../stacks/observability/) +- [apps/hello-api/metrics.go](../../apps/hello-api/metrics.go) (+ test in `main_test.go`) +- Traefik metrics: [stacks/traefik/traefik.yml](../../stacks/traefik/traefik.yml); scrape labels in [stacks/traefik/compose.yaml](../../stacks/traefik/compose.yaml) and [stacks/hello-api/compose.yaml](../../stacks/hello-api/compose.yaml) + ## What broke +_Write down errors and fixes here as you go._ + +Found while preparing this lab: +- **node-exporter: `path / is mounted on / but it is not a shared or slave mount`:** Docker Desktop's VM doesn't support `rslave`. Used `/:/host:ro`. +- **hello-api not scraped:** Prometheus' Docker discovery uses only a container's *first* network by default (`match_first_network: true`), and hello-api is on two. Fixed with `match_first_network: false`. +- **cAdvisor saw no containers (3 problems in a row):** + 1. `/var/run` on Docker Desktop is shared from macOS, so the socket was a broken symlink. Pointed cAdvisor at the socket proxy instead. + 2. The proxy answered **403**: cAdvisor needs `/info`. Allowed `INFO: 1` (still read-only). + 3. Docker 29 keeps images in containerd. Mounted `/run/containerd/containerd.sock:ro`. +- **Alloy near its memory limit** (238 of 256 MiB): raised to 384m. Limits need measuring, not guessing. +- **No alert when hello-api stopped:** a stopped container *disappears* from Docker discovery, so `up == 0` never fires. Added `ServiceMissing` with `absent(up{job="hello-api"})`. +- `curl $G/...` with options in a variable failed in zsh again. Use functions for repeated commands. + ## Lessons learned ## References + +- [Prometheus: getting started](https://prometheus.io/docs/prometheus/latest/getting_started/) +- [PromQL basics](https://prometheus.io/docs/prometheus/latest/querying/basics/) +- [Prometheus: docker_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#docker_sd_config) +- [Alerting rules](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) and [Alertmanager configuration](https://prometheus.io/docs/alerting/latest/configuration/) +- [Loki: LogQL](https://grafana.com/docs/loki/latest/query/) +- [Grafana Alloy: loki.source.docker](https://grafana.com/docs/alloy/latest/reference/components/loki/loki.source.docker/) +- [Grafana provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/) diff --git a/labs/11-observability/knowledge.md b/labs/11-observability/knowledge.md new file mode 100644 index 0000000..396aad4 --- /dev/null +++ b/labs/11-observability/knowledge.md @@ -0,0 +1,162 @@ +# Lab 11 β€” Knowledge base + +Study this **before** the hands-on part in [README.md](README.md). Add your own notes as you learn. + +## Why this matters + +So far, you've found problems by running `docker ps`, `docker logs`, and `curl`, *after* you noticed something was wrong. In production you want to: + +1. **Know before users tell you**: alerts. +2. **See what "normal" looks like**, so you notice "abnormal": dashboards and history. +3. **Find the cause quickly**: metrics show *what* and *when*; logs show *why*. + +**Monitoring** asks known questions ("is it up? is the disk full?"). **Observability** is being able to answer *new* questions from the data you collect, without shipping new code. + +## Core concepts + +### The three signals + +| Signal | What | Example | Tool here | +|---|---|---|---| +| **Metrics** | Numbers over time, cheap to store and fast to query | `http_requests_total`, CPU % | Prometheus | +| **Logs** | Text events with details | `POST /visits 500: connection refused` | Loki | +| **Traces** | One request's path through many services, with timings | API β†’ DB β†’ cache | (not in this lab: Tempo, Jaeger) | + +Typical flow: an **alert** fires on a metric β†’ the **dashboard** shows when it started and what else changed β†’ **logs** from that time explain why. + +### Prometheus: pull-based metrics + +``` +Prometheus ──every 15 s: GET /metrics──▢ target + ◀── plain text ────────────── http_requests_total{route="GET /health",code="200"} 151 +``` + +- Prometheus **pulls** (scrapes) each target. Targets just expose `/metrics`. +- If a scrape fails, Prometheus records `up = 0` for that target. That's free availability monitoring. +- Data is a **time series**: a metric name plus **labels** identify it, and each scrape adds a (time, value) sample. +- **Exporters** translate things that don't speak Prometheus: node-exporter (the Linux host), cAdvisor (containers). Traefik and hello-api expose metrics natively. + +### Metric types + +| Type | Behaves | Use for | Query with | +|---|---|---|---| +| **Counter** | Only goes up (resets to 0 on restart) | Requests, errors, visits | `rate(x[5m])`, `increase(x[1h])` | +| **Gauge** | Up and down | Memory, temperature, queue length | The value itself | +| **Histogram** | Counts observations in buckets | Latency, sizes | `histogram_quantile(0.95, …)` | + +Never graph a raw counter: it just climbs. `rate()` turns it into "per second", and handles resets. + +### Labels and cardinality + +Every unique combination of label values is a **separate time series**, costing memory and disk. hello-api labels requests by `route` (the *pattern*, like `GET /visits`), not the raw path. If `/nope`, `/nope2`, … each became a label value, an attacker or a crawler could create millions of series and take Prometheus down. The test checks unknown paths become `route="unmatched"`. + +Rules: **labels for things with few values** (method, route, status code, service). **Never** user IDs, request IDs, emails, or full URLs. Those belong in logs. + +### RED and USE + +Two checklists for "what should I graph?": + +| Method | For | Measure | +|---|---|---| +| **RED** | Services (requests) | **R**ate, **E**rrors, **D**uration | +| **USE** | Resources (CPU, disk, network) | **U**tilization, **S**aturation, **E**rrors | + +The OpsForge dashboard has RED from Traefik and hello-api, and USE from node-exporter and cAdvisor. + +### Percentiles, not averages + +An average latency of 50 ms can hide that 5% of users wait 2 seconds. **p95** means "95% of requests are faster than this". Histograms let Prometheus estimate any percentile: `histogram_quantile(0.95, sum by (le) (rate(..._bucket[5m])))`. + +### Service discovery + +Static targets (`node-exporter:9100`) are fine for fixed services. hello-api's container name changes on every deploy (lab 08), and there can be two replicas at once. **Docker service discovery** asks the Docker API which containers carry `prometheus.scrape=true`, and scrapes each one. **Relabeling** turns Docker metadata into the target address and labels. + +⚠️ With discovery, **a stopped container isn't "down", it's gone**. No target means no `up` series, so `up == 0` can't fire. Pair it with `absent(up{job="…"})`. + +### Logs with Loki + +- Loki indexes **only labels** (`project`, `service`, `container`), not the log text. It's cheap to run, and you filter text at query time (`|= "error"`). +- The same cardinality rules apply: few labels, low cardinality. +- **Alloy** discovers containers through the Docker API and ships their stdout/stderr. Apps just print logs; they don't need to know about Loki. +- **LogQL**: `{selector} |= "text" | json | line_format …`, and metric queries like `count_over_time(...[1m])`. + +### Alerting + +``` +Prometheus rule: expr true ──for: 1m──▢ FIRING ──▢ Alertmanager: group, dedupe, route ──▢ Telegram / Discord + (meanwhile: PENDING) silence, inhibit, repeat_interval +``` + +- **`for:`** avoids alerting on a single bad scrape (flapping). +- **Alertmanager** groups related alerts into one message, routes by labels (`severity`), handles **silences** during maintenance, and sends **resolved** notices. +- Good alerts are **actionable** and about **symptoms users feel** ("5% of requests fail"), not every cause ("CPU 80%"). Too many alerts and people ignore them (alert fatigue). + +### Least privilege with the Docker API + +Prometheus, Alloy, and cAdvisor need to *read* container information. Mounting `/var/run/docker.sock` would give them full control of Docker, which is root on the host (labs 05–07). The **socket proxy** exposes only read-only endpoints (`CONTAINERS`, `NETWORKS`, `INFO`; `POST: 0`) on an internal network. A request it doesn't allow gets `403 Forbidden`, which you saw when cAdvisor first asked for `/info`. + +### Provisioning: dashboards as code + +Grafana data sources and dashboards come from files in git. Rebuild the server (lab 09) and Grafana comes back identical. No "I clicked it together once" dashboards that disappear with the server. + +### Observability isn't free + +Every component uses CPU, RAM, and disk (~780 MiB RAM here). Retention (`15d` metrics, `7d` logs) and `mem_limit`s keep it bounded. Measure, then set limits: Alloy needed more than first guessed. + +## Key terms + +| Term | Meaning | +|---|---| +| Scrape | Prometheus fetching `/metrics` from a target | +| Target | Something Prometheus scrapes | +| Exporter | Adapter exposing another system's metrics | +| Time series | One metric name + one label combination over time | +| Cardinality | Number of distinct time series | +| PromQL / LogQL | Query languages of Prometheus / Loki | +| Recording / alerting rule | A query Prometheus evaluates on a schedule, to store / to alert | +| Pending / firing / resolved | Alert states | +| Silence | Temporarily muting matching alerts | + +## Commands to know + +| Command / query | What it does | +|---|---| +| `up` | Which targets are reachable | +| `rate(http_requests_total[5m])` | Requests per second | +| `sum by (code) (rate(...))` | Aggregate by a label | +| `histogram_quantile(0.95, sum by (le) (rate(x_bucket[5m])))` | p95 latency | +| `absent(up{job="x"})` | 1 when no such series exists | +| `{service="app"} \|= "error"` | Loki: log lines containing "error" | +| `promtool check config` / `check rules` | Validate Prometheus files | +| `amtool check-config` | Validate Alertmanager config | +| `docker stats --no-stream` | RAM and CPU per container | + +## Before you start, can you answer these? + +1. What's the difference between a counter and a gauge, and why do counters need `rate()`? +2. Why does hello-api label requests with the route pattern instead of the URL path? +3. What do metrics tell you that logs don't, and the other way round? +4. Why does an alert have a `for:` duration? +5. Why do Prometheus and Alloy talk to a socket proxy instead of the Docker socket? + +## After the lab, check yourself + +1. Write the PromQL for "hello-api 5xx errors per second". +2. Why didn't `up == 0` fire when hello-api was stopped, and what did? +3. What fraction of hello-api's logs were health checks? How would you reduce that? +4. Which panel would you look at first if users said "the site is slow"? +5. How much RAM did the stack use, and what would you drop on a smaller VPS? +6. What would you change to monitor the server from *outside* (if the whole VPS goes down, so does this stack)? + +## My notes + +_Your own explanations, analogies, and "aha" moments._ + +## Further reading + +- [Prometheus: overview](https://prometheus.io/docs/introduction/overview/) +- [Google SRE book: Monitoring distributed systems](https://sre.google/sre-book/monitoring-distributed-systems/) +- [The RED method](https://grafana.com/blog/2018/08/02/the-red-method-how-to-instrument-your-services/) +- [The USE method (Brendan Gregg)](https://www.brendangregg.com/usemethod.html) +- [Prometheus: instrumentation best practices](https://prometheus.io/docs/practices/instrumentation/) +- [Grafana Loki: label best practices](https://grafana.com/docs/loki/latest/get-started/labels/bp-labels/) diff --git a/stacks/README.md b/stacks/README.md index a75df03..e9b8767 100644 --- a/stacks/README.md +++ b/stacks/README.md @@ -10,6 +10,7 @@ Docker Compose stacks, one folder per stack (`stacks//compose.yaml`, plus | [gitea](gitea/) | Self-hosted Git server (`git.`), SSH on 2222 | 06 | | [uptime-kuma](uptime-kuma/) | Uptime monitoring (`status.`) | 06 | | [portainer](portainer/) | Docker management UI (`portainer.`); read-write Docker socket | 06 | +| [observability](observability/) | Prometheus, Alertmanager, Grafana (`grafana.`), Loki, Alloy, node-exporter, cAdvisor | 11 | Web-facing stacks join the shared external `proxy` network (`docker network create proxy`) and are routed by Traefik labels. diff --git a/stacks/hello-api/compose.yaml b/stacks/hello-api/compose.yaml index 4d0b821..b204572 100644 --- a/stacks/hello-api/compose.yaml +++ b/stacks/hello-api/compose.yaml @@ -15,6 +15,11 @@ services: ports: # Bound to localhost only: Docker-published ports bypass ufw (see lab 01). - "127.0.0.1:${APP_PORT:-8000}:8000" + labels: + # Scraped by Prometheus via Docker discovery (lab 11): every replica is found. + prometheus.scrape: "true" + prometheus.port: "8000" + prometheus.job: hello-api depends_on: db: condition: service_healthy diff --git a/stacks/observability/.env.example b/stacks/observability/.env.example new file mode 100644 index 0000000..62b74ee --- /dev/null +++ b/stacks/observability/.env.example @@ -0,0 +1,3 @@ +# Copy to .env (git-ignored). +DOMAIN=opsforge.localhost +GRAFANA_ADMIN_PASSWORD=change-me diff --git a/stacks/observability/alertmanager/alertmanager.yml b/stacks/observability/alertmanager/alertmanager.yml new file mode 100644 index 0000000..d30e3e6 --- /dev/null +++ b/stacks/observability/alertmanager/alertmanager.yml @@ -0,0 +1,27 @@ +# Alertmanager (lab 11): groups alerts and sends notifications. +# By default alerts are only shown in the UI. To get messages, uncomment ONE +# receiver below, put the secret in the file it names (alertmanager/secrets/ +# is git-ignored), point route.receiver at it, and restart alertmanager. +route: + receiver: ui-only + group_by: [alertname] + group_wait: 30s # wait for related alerts before the first message + group_interval: 5m # then batch new alerts in the same group + repeat_interval: 4h # remind while still firing + +receivers: + - name: ui-only + + # Telegram: create a bot with @BotFather, put its token in secrets/telegram-token, + # and use your chat id (message @userinfobot to find it). + # - name: telegram + # telegram_configs: + # - bot_token_file: /etc/alertmanager/secrets/telegram-token + # chat_id: 123456789 + # send_resolved: true + + # Discord: Server settings -> Integrations -> Webhooks, put the URL in secrets/discord-webhook. + # - name: discord + # discord_configs: + # - webhook_url_file: /etc/alertmanager/secrets/discord-webhook + # send_resolved: true diff --git a/stacks/observability/alertmanager/secrets/.gitkeep b/stacks/observability/alertmanager/secrets/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/stacks/observability/alloy/config.alloy b/stacks/observability/alloy/config.alloy new file mode 100644 index 0000000..765a4d6 --- /dev/null +++ b/stacks/observability/alloy/config.alloy @@ -0,0 +1,40 @@ +// Grafana Alloy (lab 11): ship every container's logs to Loki. +// Containers write to stdout/stderr as usual; nothing changes in the apps. + +discovery.docker "containers" { + host = "tcp://socket-proxy:2375" +} + +// Turn Docker metadata into Loki labels. Keep labels few and low-cardinality: +// project, service, container. Never request IDs or user IDs. +discovery.relabel "containers" { + targets = discovery.docker.containers.targets + + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/(.*)" + target_label = "container" + } + + rule { + source_labels = ["__meta_docker_container_label_com_docker_compose_project"] + target_label = "project" + } + + rule { + source_labels = ["__meta_docker_container_label_com_docker_compose_service"] + target_label = "service" + } +} + +loki.source.docker "containers" { + host = "tcp://socket-proxy:2375" + targets = discovery.relabel.containers.output + forward_to = [loki.write.local.receiver] +} + +loki.write "local" { + endpoint { + url = "http://loki:3100/loki/api/v1/push" + } +} diff --git a/stacks/observability/compose.yaml b/stacks/observability/compose.yaml new file mode 100644 index 0000000..9df9149 --- /dev/null +++ b/stacks/observability/compose.yaml @@ -0,0 +1,163 @@ +# Observability: metrics (Prometheus), logs (Loki + Alloy), dashboards (Grafana), +# alerts (Alertmanager) for everything on this host (lab 11). +# Prerequisites: the traefik stack, the proxy network, and .env (from .env.example). +# Usage: docker compose up -d +name: observability + +services: + # --- Metrics --------------------------------------------------------------- + prometheus: + image: prom/prometheus:v3.15.0 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=15d + - --web.external-url=https://prometheus.${DOMAIN:-opsforge.localhost} + volumes: + - ./prometheus:/etc/prometheus:ro + - prometheus-data:/prometheus + networks: [proxy, observability] + labels: + - traefik.enable=true + - traefik.http.routers.prometheus.rule=Host(`prometheus.${DOMAIN:-opsforge.localhost}`) + - traefik.http.routers.prometheus.entrypoints=websecure + - traefik.http.routers.prometheus.middlewares=dashboard-auth@file + - traefik.http.services.prometheus.loadbalancer.server.port=9090 + mem_limit: 512m + restart: unless-stopped + + alertmanager: + image: prom/alertmanager:v0.34.1 + command: + - --config.file=/etc/alertmanager/alertmanager.yml + - --storage.path=/alertmanager + - --web.external-url=https://alertmanager.${DOMAIN:-opsforge.localhost} + volumes: + - ./alertmanager:/etc/alertmanager:ro + - alertmanager-data:/alertmanager + networks: [proxy, observability] + labels: + - traefik.enable=true + - traefik.http.routers.alertmanager.rule=Host(`alertmanager.${DOMAIN:-opsforge.localhost}`) + - traefik.http.routers.alertmanager.entrypoints=websecure + - traefik.http.routers.alertmanager.middlewares=dashboard-auth@file + - traefik.http.services.alertmanager.loadbalancer.server.port=9093 + mem_limit: 128m + restart: unless-stopped + + # Host metrics: CPU, memory, disk, network. On Docker Desktop this is the + # Linux VM Docker runs in, not your Mac. + node-exporter: + image: prom/node-exporter:v1.12.1 + command: + - --path.rootfs=/host + pid: host + volumes: + # On a Linux VPS, ":ro,rslave" also picks up filesystems mounted later; + # Docker Desktop's VM doesn't support rslave, so plain read-only is used. + - /:/host:ro + networks: [observability] + mem_limit: 64m + restart: unless-stopped + + # Per-container metrics: CPU, memory, network, restarts. + cadvisor: + image: ghcr.io/google/cadvisor:v0.60.6 + command: + - --docker_only=true + - --housekeeping_interval=15s + - --store_container_labels=false + - --whitelisted_container_labels=com.docker.compose.project,com.docker.compose.service + # Container names and labels via the read-only socket proxy, not the socket. + # (Mounting /var/run also breaks on Docker Desktop: it's shared from macOS.) + - --docker=tcp://socket-proxy:2375 + # Docker 29 stores images in containerd: cAdvisor reads container details there. + - --containerd=/run/containerd/containerd.sock + privileged: true + devices: + - /dev/kmsg + volumes: + - /:/rootfs:ro + - /sys:/sys:ro + # cAdvisor is privileged (it reads every container's cgroups), so it can + # already see everything; this socket adds container metadata. + - /run/containerd/containerd.sock:/run/containerd/containerd.sock:ro + - /var/lib/docker/:/var/lib/docker:ro + networks: [observability] + mem_limit: 256m + restart: unless-stopped + + # --- Logs ------------------------------------------------------------------ + loki: + image: grafana/loki:3.7.8 + command: -config.file=/etc/loki/loki.yml + volumes: + - ./loki/loki.yml:/etc/loki/loki.yml:ro + - loki-data:/loki + networks: [observability] + mem_limit: 512m + restart: unless-stopped + + # Collects every container's logs through the Docker API and sends them to Loki. + alloy: + image: grafana/alloy:v1.20.1 + command: + - run + - --storage.path=/var/lib/alloy/data + - /etc/alloy/config.alloy + volumes: + - ./alloy/config.alloy:/etc/alloy/config.alloy:ro + - alloy-data:/var/lib/alloy/data + networks: [observability] + mem_limit: 384m # measured ~240 MiB tailing ~25 containers + restart: unless-stopped + + # Read-only Docker API for Prometheus and Alloy, instead of mounting the + # socket (root-equivalent) into them (labs 05-06). Only on the internal network. + socket-proxy: + image: tecnativa/docker-socket-proxy:v0.5.0 + environment: + CONTAINERS: 1 # list/inspect containers, read logs + NETWORKS: 1 # needed by Docker service discovery + INFO: 1 # docker info (cAdvisor checks it at startup) + POST: 0 # read-only: no create, start, stop, exec + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + networks: [observability] + mem_limit: 64m + restart: unless-stopped + + # --- Dashboards -------------------------------------------------------------- + grafana: + image: grafana/grafana:13.2.3 + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?set GRAFANA_ADMIN_PASSWORD in .env} + GF_SERVER_ROOT_URL: https://grafana.${DOMAIN:-opsforge.localhost} + GF_USERS_ALLOW_SIGN_UP: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_NEWS_NEWS_FEED_ENABLED: "false" + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana-data:/var/lib/grafana + networks: [proxy, observability] + labels: + - traefik.enable=true + - traefik.http.routers.grafana.rule=Host(`grafana.${DOMAIN:-opsforge.localhost}`) + - traefik.http.routers.grafana.entrypoints=websecure + - traefik.http.services.grafana.loadbalancer.server.port=3000 + mem_limit: 256m + restart: unless-stopped + +volumes: + prometheus-data: + alertmanager-data: + loki-data: + alloy-data: + grafana-data: + +networks: + proxy: + external: true + observability: {} diff --git a/stacks/observability/grafana/dashboards/opsforge-overview.json b/stacks/observability/grafana/dashboards/opsforge-overview.json new file mode 100644 index 0000000..37f4748 --- /dev/null +++ b/stacks/observability/grafana/dashboards/opsforge-overview.json @@ -0,0 +1,690 @@ +{ + "uid": "opsforge-overview", + "title": "OpsForge overview", + "tags": [ + "opsforge" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { + "from": "now-1h", + "to": "now" + }, + "editable": false, + "panels": [ + { + "id": 1, + "type": "row", + "title": "Overview", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 2, + "title": "Targets down", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Scrape targets Prometheus can't reach. Should be 0.", + "gridPos": { + "x": 0, + "y": 1, + "w": 6, + "h": 4 + }, + "targets": [ + { + "expr": "count(up == 0) or vector(0)", + "legendFormat": "down", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none" + } + }, + { + "id": 3, + "title": "Firing alerts", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "x": 6, + "y": 1, + "w": 6, + "h": 4 + }, + "targets": [ + { + "expr": "count(ALERTS{alertstate=\"firing\"}) or vector(0)", + "legendFormat": "firing", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none" + } + }, + { + "id": 4, + "title": "Visits recorded (since app start)", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "A business metric from hello-api. Counters reset when a container restarts; use increase() for rates.", + "gridPos": { + "x": 12, + "y": 1, + "w": 6, + "h": 4 + }, + "targets": [ + { + "expr": "sum(hello_api_visits_recorded_total) or vector(0)", + "legendFormat": "visits", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none" + } + }, + { + "id": 5, + "title": "Requests/s (all services)", + "type": "stat", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "x": 18, + "y": 1, + "w": 6, + "h": 4 + }, + "targets": [ + { + "expr": "sum(rate(traefik_service_requests_total[$__rate_interval])) or vector(0)", + "legendFormat": "req/s", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "value", + "graphMode": "none" + } + }, + { + "id": 6, + "type": "row", + "title": "Host", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 5, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 7, + "title": "CPU busy", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "x": 0, + "y": 6, + "w": 8, + "h": 7 + }, + "targets": [ + { + "expr": "1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval]))", + "legendFormat": "cpu", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 8, + "title": "Memory used", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "x": 8, + "y": 6, + "w": 8, + "h": 7 + }, + "targets": [ + { + "expr": "1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes", + "legendFormat": "memory", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 9, + "title": "Disk free", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "x": 16, + "y": 6, + "w": 8, + "h": 7 + }, + "targets": [ + { + "expr": "node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay|squashfs\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay|squashfs\"}", + "legendFormat": "{{mountpoint}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 10, + "type": "row", + "title": "Containers", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 13, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 11, + "title": "Memory by container", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "x": 0, + "y": 14, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum by (name) (container_memory_working_set_bytes{name!=\"\"})", + "legendFormat": "{{name}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 12, + "title": "CPU by container", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "CPU cores used.", + "gridPos": { + "x": 12, + "y": 14, + "w": 12, + "h": 8 + }, + "targets": [ + { + "expr": "sum by (name) (rate(container_cpu_usage_seconds_total{name!=\"\"}[$__rate_interval]))", + "legendFormat": "{{name}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 13, + "type": "row", + "title": "HTTP (Traefik and hello-api)", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 22, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 14, + "title": "Requests/s by service", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "x": 0, + "y": 23, + "w": 8, + "h": 8 + }, + "targets": [ + { + "expr": "sum by (service) (rate(traefik_service_requests_total[$__rate_interval]))", + "legendFormat": "{{service}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 15, + "title": "5xx errors/s by service", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "x": 8, + "y": 23, + "w": 8, + "h": 8 + }, + "targets": [ + { + "expr": "sum by (service) (rate(traefik_service_requests_total{code=~\"5..\"}[$__rate_interval]))", + "legendFormat": "{{service}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 16, + "title": "hello-api p95 latency by route", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "gridPos": { + "x": 16, + "y": 23, + "w": 8, + "h": 8 + }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (le, route) (rate(http_request_duration_seconds_bucket{job=\"hello-api\"}[$__rate_interval])))", + "legendFormat": "{{route}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "lineWidth": 1, + "fillOpacity": 10, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 17, + "type": "row", + "title": "Logs", + "collapsed": false, + "gridPos": { + "x": 0, + "y": 31, + "w": 24, + "h": 1 + }, + "panels": [] + }, + { + "id": 18, + "title": "hello-api logs", + "type": "logs", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "description": "", + "gridPos": { + "x": 0, + "y": 32, + "w": 24, + "h": 10 + }, + "targets": [ + { + "expr": "{project=\"hello-api\", service=\"app\"}", + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + } + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "showTime": true, + "wrapLogMessage": true, + "sortOrder": "Descending" + } + } + ], + "templating": { + "list": [] + }, + "annotations": { + "list": [] + } +} diff --git a/stacks/observability/grafana/provisioning/dashboards/dashboards.yml b/stacks/observability/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..0618a26 --- /dev/null +++ b/stacks/observability/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,9 @@ +apiVersion: 1 +providers: + - name: opsforge + folder: OpsForge + type: file + disableDeletion: true + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards diff --git a/stacks/observability/grafana/provisioning/datasources/datasources.yml b/stacks/observability/grafana/provisioning/datasources/datasources.yml new file mode 100644 index 0000000..484a2f7 --- /dev/null +++ b/stacks/observability/grafana/provisioning/datasources/datasources.yml @@ -0,0 +1,15 @@ +# Data sources, provisioned from files: no clicking, same on every rebuild. +apiVersion: 1 +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + url: http://prometheus:9090 + isDefault: true + editable: false + + - name: Loki + uid: loki + type: loki + url: http://loki:3100 + editable: false diff --git a/stacks/observability/loki/loki.yml b/stacks/observability/loki/loki.yml new file mode 100644 index 0000000..c3651f0 --- /dev/null +++ b/stacks/observability/loki/loki.yml @@ -0,0 +1,37 @@ +# Loki (lab 11): single process, files on a local volume, 7 days of logs. +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + retention_period: 168h + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + +analytics: + reporting_enabled: false diff --git a/stacks/observability/prometheus/alerts.yml b/stacks/observability/prometheus/alerts.yml new file mode 100644 index 0000000..999c78f --- /dev/null +++ b/stacks/observability/prometheus/alerts.yml @@ -0,0 +1,61 @@ +# Alert rules (lab 11). Test with: promtool check rules alerts.yml +groups: + - name: availability + rules: + - alert: TargetDown + expr: up == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "{{ $labels.job }} target {{ $labels.instance }} is down" + description: "Prometheus has not been able to scrape it for over 1 minute." + + # Containers found by Docker discovery don't go "down" when stopped: they + # disappear, and up == 0 never fires. absent() catches "nothing to scrape". + - alert: ServiceMissing + expr: absent(up{job="hello-api"}) or absent(up{job="traefik"}) + for: 1m + labels: + severity: critical + annotations: + summary: "No {{ $labels.job }} instance is running" + description: "Prometheus finds no {{ $labels.job }} container to scrape for over 1 minute." + + - alert: HelloApiHighErrorRate + expr: | + sum(rate(traefik_service_requests_total{service=~"hello-api.*", code=~"5.."}[5m])) + / sum(rate(traefik_service_requests_total{service=~"hello-api.*"}[5m])) > 0.05 + for: 5m + labels: + severity: warning + annotations: + summary: "hello-api: {{ $value | humanizePercentage }} of requests fail with 5xx" + + - name: host + rules: + - alert: HostDiskAlmostFull + expr: | + node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs"} + / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs"} < 0.10 + for: 5m + labels: + severity: warning + annotations: + summary: "Disk {{ $labels.mountpoint }} has less than 10% free" + + - alert: HostHighMemory + expr: 1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes > 0.90 + for: 5m + labels: + severity: warning + annotations: + summary: "Memory is {{ $value | humanizePercentage }} used" + + - alert: HostHighCpu + expr: 1 - avg(rate(node_cpu_seconds_total{mode="idle"}[5m])) > 0.90 + for: 10m + labels: + severity: warning + annotations: + summary: "CPU is {{ $value | humanizePercentage }} busy" diff --git a/stacks/observability/prometheus/prometheus.yml b/stacks/observability/prometheus/prometheus.yml new file mode 100644 index 0000000..4b24568 --- /dev/null +++ b/stacks/observability/prometheus/prometheus.yml @@ -0,0 +1,51 @@ +# Prometheus (lab 11): what to scrape, how often, and where alerts go. +global: + scrape_interval: 15s + evaluation_interval: 15s + +rule_files: + - /etc/prometheus/alerts.yml + +alerting: + alertmanagers: + - static_configs: + - targets: [alertmanager:9093] + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: [localhost:9090] + + - job_name: node + static_configs: + - targets: [node-exporter:9100] + + - job_name: cadvisor + static_configs: + - targets: [cadvisor:8080] + + # Any container with the label prometheus.scrape=true, on the proxy network. + # Finds every replica (container names change on each deploy, lab 08). + - job_name: docker + docker_sd_configs: + - host: tcp://socket-proxy:2375 + refresh_interval: 15s + # Default true = only a container's first network. hello-api is on two + # (its own + proxy); we need one target per network to keep "proxy" below. + match_first_network: false + filters: + - name: label + values: [prometheus.scrape=true] + relabel_configs: + - source_labels: [__meta_docker_network_name] + regex: proxy + action: keep + - source_labels: [__meta_docker_network_ip, __meta_docker_container_label_prometheus_port] + regex: (.+);(.+) + replacement: $1:$2 + target_label: __address__ + - source_labels: [__meta_docker_container_label_prometheus_job] + target_label: job + - source_labels: [__meta_docker_container_name] + regex: /(.*) + target_label: container diff --git a/stacks/traefik/compose.yaml b/stacks/traefik/compose.yaml index 05ad2ee..79ee1f8 100644 --- a/stacks/traefik/compose.yaml +++ b/stacks/traefik/compose.yaml @@ -25,6 +25,10 @@ services: - traefik.http.routers.dashboard.entrypoints=websecure - traefik.http.routers.dashboard.service=api@internal - traefik.http.routers.dashboard.middlewares=dashboard-auth@file + # Scraped by Prometheus (lab 11) + - prometheus.scrape=true + - prometheus.port=8082 + - prometheus.job=traefik restart: unless-stopped networks: diff --git a/stacks/traefik/traefik.yml b/stacks/traefik/traefik.yml index 10cbf7a..4d2e201 100644 --- a/stacks/traefik/traefik.yml +++ b/stacks/traefik/traefik.yml @@ -14,6 +14,8 @@ entryPoints: address: ":443" http: tls: {} # every router on this entry point uses TLS + metrics: + address: ":8082" # Prometheus metrics; not published, scraped over the proxy network providers: docker: @@ -30,3 +32,10 @@ log: level: INFO accessLog: {} + +# Request rate, errors, and latency per router and service (lab 11). +metrics: + prometheus: + entryPoint: metrics + addRoutersLabels: true + addServicesLabels: true